{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0125 0.025  0.05   0.1    0.2    0.4    0.8   ]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression, Ridge\n",
    "from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error, r2_score, roc_auc_score, precision_recall_curve, auc\n",
    "\n",
    "rates = 2**np.arange(7)/80\n",
    "print(rates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_inputs(sm):\n",
    "    seq_len = 220\n",
    "    sm = sm.split()\n",
    "    if len(sm)>218:\n",
    "        print('SMILES is too long ({:d})'.format(len(sm)))\n",
    "        sm = sm[:109]+sm[-109:]\n",
    "    ids = [vocab.stoi.get(token, unk_index) for token in sm]\n",
    "    ids = [sos_index] + ids + [eos_index]\n",
    "    seg = [1]*len(ids)\n",
    "    padding = [pad_index]*(seq_len - len(ids))\n",
    "    ids.extend(padding), seg.extend(padding)\n",
    "    return ids, seg\n",
    "\n",
    "def get_array(smiles):\n",
    "    x_id, x_seg = [], []\n",
    "    for sm in smiles:\n",
    "        a,b = get_inputs(sm)\n",
    "        x_id.append(a)\n",
    "        x_seg.append(b)\n",
    "    return torch.tensor(x_id), torch.tensor(x_seg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit import Chem\n",
    "from rdkit.Chem import AllChem\n",
    "\n",
    "def bit2np(bitvector):\n",
    "    bitstring = bitvector.ToBitString()\n",
    "    intmap = map(int, bitstring)\n",
    "    return np.array(list(intmap))\n",
    "\n",
    "def extract_morgan(smiles, targets):\n",
    "    x,X,y = [],[],[]\n",
    "    for sm,target in zip(smiles,targets):\n",
    "        mol = Chem.MolFromSmiles(sm)\n",
    "        if mol is None:\n",
    "            print(sm)\n",
    "            continue\n",
    "        fp = AllChem.GetMorganFingerprintAsBitVect(mol, 2, 1024) # Morgan (Similar to ECFP4)\n",
    "        x.append(sm)\n",
    "        X.append(bit2np(fp))\n",
    "        y.append(target)\n",
    "    return x,np.array(X),np.array(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST, RNN, BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total parameters: 4245037\n",
      "Total parameters: 4713517\n",
      "Total parameters: 6330368\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from pretrain_trfm import TrfmSeq2seq\n",
    "from pretrain_rnn import RNNSeq2Seq\n",
    "from bert import BERT\n",
    "from build_vocab import WordVocab\n",
    "from utils import split\n",
    "\n",
    "pad_index = 0\n",
    "unk_index = 1\n",
    "eos_index = 2\n",
    "sos_index = 3\n",
    "mask_index = 4\n",
    "\n",
    "vocab = WordVocab.load_vocab('data/vocab.pkl')\n",
    "\n",
    "trfm = TrfmSeq2seq(len(vocab), 256, len(vocab), 3)\n",
    "trfm.load_state_dict(torch.load('.save/trfm_12_23000.pkl'))\n",
    "trfm.eval()\n",
    "print('Total parameters:', sum(p.numel() for p in trfm.parameters()))\n",
    "\n",
    "rnn = RNNSeq2Seq(len(vocab), 256, len(vocab), 3)\n",
    "rnn.load_state_dict(torch.load('.save/seq2seq_1.pkl'))\n",
    "rnn.eval()\n",
    "print('Total parameters:', sum(p.numel() for p in rnn.parameters()))\n",
    "\n",
    "bert = BERT(len(vocab), hidden=256, n_layers=8, attn_heads=8, dropout=0)\n",
    "bert.load_state_dict(torch.load('../result/chembl/ep00_it010000.pkl'))\n",
    "bert.eval()\n",
    "print('Total parameters:', sum(p.numel() for p in bert.parameters()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_regression(X, y, rate, n_repeats, model='ridge'):\n",
    "    r2, rmse = np.empty(n_repeats), np.empty(n_repeats)\n",
    "    for i in range(n_repeats):\n",
    "        if model=='ridge':\n",
    "            reg = Ridge()\n",
    "        elif model=='rf':\n",
    "            reg = RandomForestRegressor(n_estimators=10)\n",
    "        else:\n",
    "            raise ValueError('Model \"{}\" is invalid. Specify \"ridge\" or \"rf\".'.format(model))\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1-rate)\n",
    "        reg.fit(X_train, y_train)\n",
    "        y_pred = reg.predict(X_test)\n",
    "        r2[i] = r2_score(y_pred, y_test)\n",
    "        rmse[i] = mean_squared_error(y_pred, y_test)**0.5\n",
    "    ret = {}\n",
    "    ret['r2 mean'] = np.mean(r2)\n",
    "    ret['r2 std'] = np.std(r2)\n",
    "    ret['rmse mean'] = np.mean(rmse)\n",
    "    ret['rmse std'] = np.std(rmse)\n",
    "    return ret\n",
    "\n",
    "def evaluate_classification(X, y, rate, n_repeats, model='ridge'):\n",
    "    roc_aucs, prc_aucs = np.empty(n_repeats), np.empty(n_repeats)\n",
    "    for i in range(n_repeats):\n",
    "        if model=='ridge':\n",
    "            clf = LogisticRegression(penalty='l2', solver='lbfgs', max_iter=1000)\n",
    "        elif model=='rf':\n",
    "            clf = RandomForestClassifier(n_estimators=10)\n",
    "        else:\n",
    "            raise ValueError('Model \"{}\" is invalid. Specify \"ridge\" or \"rf\".'.format(model))\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1-rate, stratify=y)\n",
    "        clf.fit(X_train, y_train)\n",
    "        y_score = clf.predict_proba(X_test)\n",
    "        roc_aucs[i] = roc_auc_score(y_test, y_score[:,1])\n",
    "        precision, recall, thresholds = precision_recall_curve(y_test, y_score[:,1])\n",
    "        prc_aucs[i] = auc(recall, precision)\n",
    "    ret = {}\n",
    "    ret['roc_auc mean'] = np.mean(roc_aucs)\n",
    "    ret['roc_auc std'] = np.std(roc_aucs)\n",
    "    ret['prc_auc mean'] = np.mean(prc_aucs)\n",
    "    ret['prc_auc std'] = np.std(prc_aucs)\n",
    "    \n",
    "    return ret\n",
    "\n",
    "def evaluate_classification_multi(X, rate, n_repeats, model='ridge'):\n",
    "    roc_aucs, prc_aucs = np.empty(n_repeats), np.empty(n_repeats)\n",
    "    for i in range(n_repeats):\n",
    "        _roc_aucs, _prc_aucs = np.empty(len(KEYS)), np.empty(len(KEYS))\n",
    "        for j,key in enumerate(KEYS):\n",
    "            X_dr = X[df[key].notna()]\n",
    "            y_dr = df[key].dropna().values\n",
    "            if model=='ridge':\n",
    "                clf = LogisticRegression(penalty='l2', solver='lbfgs', max_iter=1000)\n",
    "            elif model=='rf':\n",
    "                clf = RandomForestClassifier(n_estimators=10)\n",
    "            else:\n",
    "                raise ValueError('Model \"{}\" is invalid. Specify \"ridge\" or \"rf\".'.format(model))\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X_dr, y_dr, test_size=1-rate, stratify=y_dr)\n",
    "            clf.fit(X_train, y_train)\n",
    "            y_score = clf.predict_proba(X_test)\n",
    "            _roc_aucs[j] = roc_auc_score(y_test, y_score[:,1])\n",
    "            precision, recall, thresholds = precision_recall_curve(y_test, y_score[:,1])\n",
    "            _prc_aucs[j] = auc(recall, precision)\n",
    "        roc_aucs[i] = np.mean(_roc_aucs)\n",
    "        prc_aucs[i] = np.mean(_prc_aucs)\n",
    "    ret = {}\n",
    "    ret['roc_auc mean'] = np.mean(roc_aucs)\n",
    "    ret['roc_auc std'] = np.mean(np.std(roc_aucs, axis=0))\n",
    "    ret['prc_auc mean'] = np.mean(prc_aucs)\n",
    "    ret['prc_auc std'] = np.mean(np.std(prc_aucs, axis=0))\n",
    "    \n",
    "    return ret"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ESOL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1128, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Compound ID</th>\n",
       "      <th>ESOL predicted log solubility in mols per litre</th>\n",
       "      <th>Minimum Degree</th>\n",
       "      <th>Molecular Weight</th>\n",
       "      <th>Number of H-Bond Donors</th>\n",
       "      <th>Number of Rings</th>\n",
       "      <th>Number of Rotatable Bonds</th>\n",
       "      <th>Polar Surface Area</th>\n",
       "      <th>measured log solubility in mols per litre</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Amigdalin</td>\n",
       "      <td>-0.974</td>\n",
       "      <td>1</td>\n",
       "      <td>457.432</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>202.32</td>\n",
       "      <td>-0.77</td>\n",
       "      <td>OCC3OC(OCC2OC(OC(C#N)c1ccccc1)C(O)C(O)C2O)C(O)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Fenfuram</td>\n",
       "      <td>-2.885</td>\n",
       "      <td>1</td>\n",
       "      <td>201.225</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>42.24</td>\n",
       "      <td>-3.30</td>\n",
       "      <td>Cc1occc1C(=O)Nc2ccccc2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>citral</td>\n",
       "      <td>-2.579</td>\n",
       "      <td>1</td>\n",
       "      <td>152.237</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>17.07</td>\n",
       "      <td>-2.06</td>\n",
       "      <td>CC(C)=CCCC(C)=CC(=O)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Picene</td>\n",
       "      <td>-6.618</td>\n",
       "      <td>2</td>\n",
       "      <td>278.354</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-7.87</td>\n",
       "      <td>c1ccc2c(c1)ccc3c2ccc4c5ccccc5ccc43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Thiophene</td>\n",
       "      <td>-2.232</td>\n",
       "      <td>2</td>\n",
       "      <td>84.143</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-1.33</td>\n",
       "      <td>c1ccsc1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Compound ID  ESOL predicted log solubility in mols per litre  \\\n",
       "0   Amigdalin                                           -0.974   \n",
       "1    Fenfuram                                           -2.885   \n",
       "2      citral                                           -2.579   \n",
       "3      Picene                                           -6.618   \n",
       "4   Thiophene                                           -2.232   \n",
       "\n",
       "   Minimum Degree  Molecular Weight  Number of H-Bond Donors  Number of Rings  \\\n",
       "0               1           457.432                        7                3   \n",
       "1               1           201.225                        1                2   \n",
       "2               1           152.237                        0                0   \n",
       "3               2           278.354                        0                5   \n",
       "4               2            84.143                        0                1   \n",
       "\n",
       "   Number of Rotatable Bonds  Polar Surface Area  \\\n",
       "0                          7              202.32   \n",
       "1                          2               42.24   \n",
       "2                          4               17.07   \n",
       "3                          0                0.00   \n",
       "4                          0                0.00   \n",
       "\n",
       "   measured log solubility in mols per litre  \\\n",
       "0                                      -0.77   \n",
       "1                                      -3.30   \n",
       "2                                      -2.06   \n",
       "3                                      -7.87   \n",
       "4                                      -1.33   \n",
       "\n",
       "                                              smiles  \n",
       "0  OCC3OC(OCC2OC(OC(C#N)c1ccccc1)C(O)C(O)C2O)C(O)...  \n",
       "1                             Cc1occc1C(=O)Nc2ccccc2  \n",
       "2                               CC(C)=CCCC(C)=CC(=O)  \n",
       "3                 c1ccc2c(c1)ccc3c2ccc4c5ccccc5ccc43  \n",
       "4                                            c1ccsc1  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/esol.csv')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1128 molecules. It will take a little time.\n",
      "(1128, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, xseg = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -0.016917272080751788, 'r2 std': 0.28445611721327063, 'rmse mean': 1.68286472585094, 'rmse std': 0.17279920056389367}\n",
      "0.025 {'r2 mean': 0.35883501823070374, 'r2 std': 0.30318971043659604, 'rmse mean': 1.4164727051500527, 'rmse std': 0.14708818597107423}\n",
      "0.05 {'r2 mean': 0.5472114844526931, 'r2 std': 0.15613272980091045, 'rmse mean': 1.2377030066942978, 'rmse std': 0.12989301476190754}\n",
      "0.1 {'r2 mean': 0.714845103452148, 'r2 std': 0.03425302510240255, 'rmse mean': 1.0763825582182052, 'rmse std': 0.04250459782422364}\n",
      "0.2 {'r2 mean': 0.767505102985931, 'r2 std': 0.020871028120941132, 'rmse mean': 0.9755392634335202, 'rmse std': 0.0368682694511301}\n",
      "0.4 {'r2 mean': 0.8144792673805592, 'r2 std': 0.015650342620878614, 'rmse mean': 0.880103964063793, 'rmse std': 0.025140734100883553}\n",
      "0.8 {'r2 mean': 0.8454934309042688, 'r2 std': 0.01904684755228456, 'rmse mean': 0.7803270370758082, 'rmse std': 0.03793611883438559}\n",
      "1.1499133229266594\n"
     ]
    }
   ],
   "source": [
    "# Ridge\n",
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['measured log solubility in mols per litre'].values, rate, 3, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -4.459816256698875, 'r2 std': 0.6648386152532737, 'rmse mean': 1.9137749181047363, 'rmse std': 0.015699717009540537}\n",
      "0.025 {'r2 mean': -1.4414285640587081, 'r2 std': 0.11655602661926534, 'rmse mean': 1.8693504649998676, 'rmse std': 0.13704560946254563}\n",
      "0.05 {'r2 mean': -0.2748806108731816, 'r2 std': 0.20370582769743728, 'rmse mean': 1.5039636504471061, 'rmse std': 0.014559565137251096}\n",
      "0.1 {'r2 mean': 0.13684260527155068, 'r2 std': 0.07393313486727338, 'rmse mean': 1.3632488441299737, 'rmse std': 0.03636275908090281}\n",
      "0.2 {'r2 mean': 0.49379057046794583, 'r2 std': 0.0670172000511483, 'rmse mean': 1.2032123375855215, 'rmse std': 0.07689830497991801}\n",
      "0.4 {'r2 mean': 0.4189423651406143, 'r2 std': 0.11763353838372842, 'rmse mean': 1.1955876554494487, 'rmse std': 0.05314417062116705}\n",
      "0.8 {'r2 mean': 0.6642121376132779, 'r2 std': 0.06488457404391562, 'rmse mean': 0.9983841889523035, 'rmse std': 0.07474660046466702}\n",
      "1.4353602942384227\n"
     ]
    }
   ],
   "source": [
    "# RF\n",
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['measured log solubility in mols per litre'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1128 1128\n"
     ]
    }
   ],
   "source": [
    "x,X,y = extract_morgan(df['smiles'].values,df['measured log solubility in mols per litre'].values)\n",
    "print(len(X), len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -7.112291424309731, 'r2 std': 3.9614568881595815, 'rmse mean': 2.051524504621592, 'rmse std': 0.14111535064331143}\n",
      "0.025 {'r2 mean': -2.9313044603669867, 'r2 std': 1.42614208549583, 'rmse mean': 1.8999503850247605, 'rmse std': 0.09525535401564483}\n",
      "0.05 {'r2 mean': -1.1514675985667169, 'r2 std': 0.6574123610624654, 'rmse mean': 1.7411075809377394, 'rmse std': 0.06919208580200438}\n",
      "0.1 {'r2 mean': -0.03523113995012293, 'r2 std': 0.22019809077710215, 'rmse mean': 1.630029844110355, 'rmse std': 0.058007942131308665}\n",
      "0.2 {'r2 mean': 0.2491544547601056, 'r2 std': 0.07321279094025546, 'rmse mean': 1.511491518015885, 'rmse std': 0.04736301143576525}\n",
      "0.4 {'r2 mean': 0.43737017968809805, 'r2 std': 0.04821233391594162, 'rmse mean': 1.4669539979988429, 'rmse std': 0.037046310420938505}\n",
      "0.8 {'r2 mean': 0.5245353262956726, 'r2 std': 0.0726333170037766, 'rmse mean': 1.4426095440586169, 'rmse std': 0.07277782705049546}\n",
      "1.6776667678239703\n"
     ]
    }
   ],
   "source": [
    "# Ridge\n",
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['measured log solubility in mols per litre'].values, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -12.016444994233252, 'r2 std': 9.729964930306064, 'rmse mean': 2.138601600779755, 'rmse std': 0.147015954525649}\n",
      "0.025 {'r2 mean': -6.167988069272901, 'r2 std': 4.952276810017011, 'rmse mean': 2.0234364092862913, 'rmse std': 0.1210772203410297}\n",
      "0.05 {'r2 mean': -2.2624924101844752, 'r2 std': 1.1836957868937854, 'rmse mean': 1.9056779142105966, 'rmse std': 0.11276788891736504}\n",
      "0.1 {'r2 mean': -0.7828317481431022, 'r2 std': 0.5487326199251339, 'rmse mean': 1.7092724641702024, 'rmse std': 0.06679520385474566}\n",
      "0.2 {'r2 mean': -0.19344600401293124, 'r2 std': 0.2515309494420422, 'rmse mean': 1.580382228654446, 'rmse std': 0.0525028922425431}\n",
      "0.4 {'r2 mean': 0.18086868280677562, 'r2 std': 0.07587050090077511, 'rmse mean': 1.4287717447635235, 'rmse std': 0.04569162894215761}\n",
      "0.8 {'r2 mean': 0.4155386935245242, 'r2 std': 0.07735464391676009, 'rmse mean': 1.2522866119744356, 'rmse std': 0.07429337803883881}\n",
      "1.7197755676913216\n"
     ]
    }
   ],
   "source": [
    "# RF\n",
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['measured log solubility in mols per litre'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1128 molecules. It will take a little time.\n",
      "(1128, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -0.7302114027803807, 'r2 std': 1.1976965856717712, 'rmse mean': 1.8304269250989862, 'rmse std': 0.1925352511502739}\n",
      "0.025 {'r2 mean': 0.16553541510652572, 'r2 std': 0.17758176895011527, 'rmse mean': 1.6323537927896254, 'rmse std': 0.1074456748026208}\n",
      "0.05 {'r2 mean': 0.4464266568430018, 'r2 std': 0.11078271744231444, 'rmse mean': 1.433532154715028, 'rmse std': 0.09717373290635564}\n",
      "0.1 {'r2 mean': 0.6019486324245832, 'r2 std': 0.027260595949040584, 'rmse mean': 1.2180057244129705, 'rmse std': 0.05232456679658844}\n",
      "0.2 {'r2 mean': 0.6987199512527263, 'r2 std': 0.027830776631066822, 'rmse mean': 1.0709738498031804, 'rmse std': 0.04300469384433809}\n",
      "0.4 {'r2 mean': 0.7586363703604122, 'r2 std': 0.01445926991912607, 'rmse mean': 0.9614645031146972, 'rmse std': 0.02129014797377028}\n",
      "0.8 {'r2 mean': 0.7986687890714661, 'r2 std': 0.02621789276029056, 'rmse mean': 0.8668650570338817, 'rmse std': 0.04463539771385796}\n",
      "1.287660286709767\n"
     ]
    }
   ],
   "source": [
    "# Ridge\n",
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['measured log solubility in mols per litre'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -4.831932864813778, 'r2 std': 3.3438846898601997, 'rmse mean': 2.0224299840297073, 'rmse std': 0.1653792442501378}\n",
      "0.025 {'r2 mean': -1.3849654520000267, 'r2 std': 0.8144642598098787, 'rmse mean': 1.7330485715281831, 'rmse std': 0.0790456336643324}\n",
      "0.05 {'r2 mean': -0.816816032791983, 'r2 std': 0.8896262817562837, 'rmse mean': 1.5900696496137023, 'rmse std': 0.08482219358053204}\n",
      "0.1 {'r2 mean': 0.010541079113252406, 'r2 std': 0.18271846473094092, 'rmse mean': 1.4287861799876074, 'rmse std': 0.05687836583398255}\n",
      "0.2 {'r2 mean': 0.2741794796412881, 'r2 std': 0.1294463209368399, 'rmse mean': 1.289459970032841, 'rmse std': 0.04928484127185311}\n",
      "0.4 {'r2 mean': 0.5160127863209739, 'r2 std': 0.03261856623887599, 'rmse mean': 1.132976621357091, 'rmse std': 0.029886109238182374}\n",
      "0.8 {'r2 mean': 0.6480188048361099, 'r2 std': 0.0626650478502643, 'rmse mean': 0.9965907946324327, 'rmse std': 0.0770624886976142}\n",
      "1.4561945387402235\n"
     ]
    }
   ],
   "source": [
    "# RF\n",
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['measured log solubility in mols per litre'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FreeSolv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(642, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iupac</th>\n",
       "      <th>smiles</th>\n",
       "      <th>expt</th>\n",
       "      <th>calc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4-methoxy-N,N-dimethyl-benzamide</td>\n",
       "      <td>CN(C)C(=O)c1ccc(cc1)OC</td>\n",
       "      <td>-11.01</td>\n",
       "      <td>-9.625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>methanesulfonyl chloride</td>\n",
       "      <td>CS(=O)(=O)Cl</td>\n",
       "      <td>-4.87</td>\n",
       "      <td>-6.219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3-methylbut-1-ene</td>\n",
       "      <td>CC(C)C=C</td>\n",
       "      <td>1.83</td>\n",
       "      <td>2.452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2-ethylpyrazine</td>\n",
       "      <td>CCc1cnccn1</td>\n",
       "      <td>-5.45</td>\n",
       "      <td>-5.809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>heptan-1-ol</td>\n",
       "      <td>CCCCCCCO</td>\n",
       "      <td>-4.21</td>\n",
       "      <td>-2.917</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              iupac                  smiles   expt   calc\n",
       "0  4-methoxy-N,N-dimethyl-benzamide  CN(C)C(=O)c1ccc(cc1)OC -11.01 -9.625\n",
       "1          methanesulfonyl chloride            CS(=O)(=O)Cl  -4.87 -6.219\n",
       "2                 3-methylbut-1-ene                CC(C)C=C   1.83  2.452\n",
       "3                   2-ethylpyrazine              CCc1cnccn1  -5.45 -5.809\n",
       "4                       heptan-1-ol                CCCCCCCO  -4.21 -2.917"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/freesolv.csv')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 642 molecules. It will take a little time.\n",
      "(642, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, xseg = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -5.419072259850387, 'r2 std': 11.119887552440943, 'rmse mean': 3.5683797540492725, 'rmse std': 0.3074231552810539}\n",
      "0.025 {'r2 mean': -0.4916844262479228, 'r2 std': 0.9973777360902092, 'rmse mean': 3.098453268845545, 'rmse std': 0.22360289869118927}\n",
      "0.05 {'r2 mean': 0.16093667747191703, 'r2 std': 0.2528491113213987, 'rmse mean': 2.7641210087245276, 'rmse std': 0.26476244734029486}\n",
      "0.1 {'r2 mean': 0.43270855238325695, 'r2 std': 0.22774472447582098, 'rmse mean': 2.393614179208929, 'rmse std': 0.2344036372289845}\n",
      "0.2 {'r2 mean': 0.6839657210336997, 'r2 std': 0.05818668520473047, 'rmse mean': 2.0081054907567606, 'rmse std': 0.09725161934661461}\n",
      "0.4 {'r2 mean': 0.7515761554535734, 'r2 std': 0.06556675440000335, 'rmse mean': 1.7750997724178277, 'rmse std': 0.1686774206992771}\n",
      "0.8 {'r2 mean': 0.8124092346749183, 'r2 std': 0.045210582979291876, 'rmse mean': 1.5569337222913022, 'rmse std': 0.14916886612949543}\n",
      "2.4521010280420237\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['expt'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -15.83643493661664, 'r2 std': 14.276631258130237, 'rmse mean': 3.9873915810067637, 'rmse std': 0.480122571249266}\n",
      "0.025 {'r2 mean': -4.663941047503971, 'r2 std': 2.2798546861776336, 'rmse mean': 3.49113987856476, 'rmse std': 0.18121601138874593}\n",
      "0.05 {'r2 mean': -1.5358474351853368, 'r2 std': 0.9847605640290039, 'rmse mean': 3.131805783418168, 'rmse std': 0.20559241457826188}\n",
      "0.1 {'r2 mean': -0.8568804225652539, 'r2 std': 0.5248621020622593, 'rmse mean': 2.9260194772307404, 'rmse std': 0.1462064811467721}\n",
      "0.2 {'r2 mean': -0.10578039381155177, 'r2 std': 0.32896797877457495, 'rmse mean': 2.6608300109853404, 'rmse std': 0.22975741206435188}\n",
      "0.4 {'r2 mean': 0.3601586432401004, 'r2 std': 0.17369016095755313, 'rmse mean': 2.252354960993647, 'rmse std': 0.1939118821250415}\n",
      "0.8 {'r2 mean': 0.4363711536792211, 'r2 std': 0.33043614157365037, 'rmse mean': 2.106211247143961, 'rmse std': 0.4194958032160875}\n",
      "2.936536134191911\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['expt'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "642 642\n"
     ]
    }
   ],
   "source": [
    "x,X,y = extract_morgan(df['smiles'].values, df['expt'].values)\n",
    "print(len(X), len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -19.806937947318453, 'r2 std': 18.125876936002978, 'rmse mean': 3.7694316255306775, 'rmse std': 0.26741836331005814}\n",
      "0.025 {'r2 mean': -5.407795870497319, 'r2 std': 3.9592537120356974, 'rmse mean': 3.5339880308451272, 'rmse std': 0.24014060982043908}\n",
      "0.05 {'r2 mean': -1.5426876447771536, 'r2 std': 0.8733405734144755, 'rmse mean': 3.1223667833646322, 'rmse std': 0.2266169968220508}\n",
      "0.1 {'r2 mean': -0.7831268229094809, 'r2 std': 0.4762345272271363, 'rmse mean': 2.949119660228903, 'rmse std': 0.17755903388733235}\n",
      "0.2 {'r2 mean': 0.16182201738539898, 'r2 std': 0.24678156316176994, 'rmse mean': 2.490817907760662, 'rmse std': 0.19107510225008514}\n",
      "0.4 {'r2 mean': 0.5595546940638719, 'r2 std': 0.09188494297812207, 'rmse mean': 2.1040465201630196, 'rmse std': 0.17292239122119335}\n",
      "0.8 {'r2 mean': 0.6636470992763959, 'r2 std': 0.14816731482564519, 'rmse mean': 1.9335315571898957, 'rmse std': 0.31085798069342413}\n",
      "2.8433288692975593\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['expt'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -38.297983005758354, 'r2 std': 67.21535921942503, 'rmse mean': 3.9347980408052114, 'rmse std': 0.3454182477106391}\n",
      "0.025 {'r2 mean': -6.569210834309546, 'r2 std': 6.88908027421144, 'rmse mean': 3.7079676202016727, 'rmse std': 0.2678314921865409}\n",
      "0.05 {'r2 mean': -2.768559905382643, 'r2 std': 1.8453483492460294, 'rmse mean': 3.432355740214283, 'rmse std': 0.21401191836402833}\n",
      "0.1 {'r2 mean': -1.1762655535064546, 'r2 std': 0.9235381770166953, 'rmse mean': 3.1131444091262797, 'rmse std': 0.2622272055293279}\n",
      "0.2 {'r2 mean': -0.28315613240902593, 'r2 std': 0.4044988324853556, 'rmse mean': 2.8878677496399248, 'rmse std': 0.15768498866094302}\n",
      "0.4 {'r2 mean': 0.22768761383358566, 'r2 std': 0.16285063489880922, 'rmse mean': 2.5405261241759804, 'rmse std': 0.1837205392675302}\n",
      "0.8 {'r2 mean': 0.3585302620078278, 'r2 std': 0.30445767785829186, 'rmse mean': 2.3217222284144907, 'rmse std': 0.5096730139110354}\n",
      "3.1340545589396918\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['expt'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 642 molecules. It will take a little time.\n",
      "(642, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -3.297858883125567, 'r2 std': 4.958118701502974, 'rmse mean': 3.769016053646291, 'rmse std': 0.43453503898815715}\n",
      "0.025 {'r2 mean': -0.8313672318040192, 'r2 std': 0.8484212730434824, 'rmse mean': 3.5278499844834874, 'rmse std': 0.3772000471154759}\n",
      "0.05 {'r2 mean': -0.11212998887660301, 'r2 std': 0.34430375894133874, 'rmse mean': 3.271884563285487, 'rmse std': 0.22834212124170772}\n",
      "0.1 {'r2 mean': 0.17971088466710905, 'r2 std': 0.24897384744943743, 'rmse mean': 2.989791834583978, 'rmse std': 0.25895843806835417}\n",
      "0.2 {'r2 mean': 0.45263413677738945, 'r2 std': 0.0978402927015076, 'rmse mean': 2.5157562408731278, 'rmse std': 0.11046952495601861}\n",
      "0.4 {'r2 mean': 0.559538848078754, 'r2 std': 0.10458808818083644, 'rmse mean': 2.216382736835478, 'rmse std': 0.1552691529678854}\n",
      "0.8 {'r2 mean': 0.6852691059904626, 'r2 std': 0.0834839340218922, 'rmse mean': 1.8797468652615112, 'rmse std': 0.19626167579523413}\n",
      "2.8814897541384803\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['expt'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -11.132670932948779, 'r2 std': 9.03921671133093, 'rmse mean': 3.8225839824964742, 'rmse std': 0.28197211936454236}\n",
      "0.025 {'r2 mean': -4.150935873603734, 'r2 std': 2.6692155174839836, 'rmse mean': 3.5286659073076705, 'rmse std': 0.14758278552110923}\n",
      "0.05 {'r2 mean': -3.9985826214595557, 'r2 std': 3.028715009535766, 'rmse mean': 3.441469685642859, 'rmse std': 0.15717345335833768}\n",
      "0.1 {'r2 mean': -0.8395957587425837, 'r2 std': 0.5282528464371651, 'rmse mean': 3.095760597003099, 'rmse std': 0.1469729393498996}\n",
      "0.2 {'r2 mean': -0.2605670052415508, 'r2 std': 0.3134294885046078, 'rmse mean': 2.7964627289647384, 'rmse std': 0.14177218348161114}\n",
      "0.4 {'r2 mean': 0.07179809754055823, 'r2 std': 0.29871169765938305, 'rmse mean': 2.5341019759536088, 'rmse std': 0.22069016702233307}\n",
      "0.8 {'r2 mean': 0.40811428149654594, 'r2 std': 0.15067486941786615, 'rmse mean': 2.0984406939426816, 'rmse std': 0.1955044502098859}\n",
      "3.045355081615875\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['expt'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lipo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4200, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CMPD_CHEMBLID</th>\n",
       "      <th>exp</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CHEMBL596271</td>\n",
       "      <td>3.54</td>\n",
       "      <td>Cn1c(CN2CCN(CC2)c3ccc(Cl)cc3)nc4ccccc14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CHEMBL1951080</td>\n",
       "      <td>-1.18</td>\n",
       "      <td>COc1cc(OC)c(cc1NC(=O)CSCC(=O)O)S(=O)(=O)N2C(C)...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CHEMBL1771</td>\n",
       "      <td>3.69</td>\n",
       "      <td>COC(=O)[C@@H](N1CCc2sccc2C1)c3ccccc3Cl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CHEMBL234951</td>\n",
       "      <td>3.37</td>\n",
       "      <td>OC[C@H](O)CN1C(=O)C(Cc2ccccc12)NC(=O)c3cc4cc(C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CHEMBL565079</td>\n",
       "      <td>3.10</td>\n",
       "      <td>Cc1cccc(C[C@H](NC(=O)c2cc(nn2C)C(C)(C)C)C(=O)N...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CMPD_CHEMBLID   exp                                             smiles\n",
       "0   CHEMBL596271  3.54            Cn1c(CN2CCN(CC2)c3ccc(Cl)cc3)nc4ccccc14\n",
       "1  CHEMBL1951080 -1.18  COc1cc(OC)c(cc1NC(=O)CSCC(=O)O)S(=O)(=O)N2C(C)...\n",
       "2     CHEMBL1771  3.69             COC(=O)[C@@H](N1CCc2sccc2C1)c3ccccc3Cl\n",
       "3   CHEMBL234951  3.37  OC[C@H](O)CN1C(=O)C(Cc2ccccc12)NC(=O)c3cc4cc(C...\n",
       "4   CHEMBL565079  3.10  Cc1cccc(C[C@H](NC(=O)c2cc(nn2C)C(C)(C)C)C(=O)N..."
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/lipo.csv')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (251)\n",
      "SMILES is too long (267)\n",
      "There are 4200 molecules. It will take a little time.\n",
      "(4200, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, xseg = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -0.7834429377596105, 'r2 std': 0.37763417022627505, 'rmse mean': 1.4622859140861015, 'rmse std': 0.08558440785165533}\n",
      "0.025 {'r2 mean': -0.6153768059314121, 'r2 std': 0.1773000569518779, 'rmse mean': 1.3403985844688862, 'rmse std': 0.078152211272996}\n",
      "0.05 {'r2 mean': -0.4515718444250784, 'r2 std': 0.16767094345980504, 'rmse mean': 1.303541689608665, 'rmse std': 0.043953874444653845}\n",
      "0.1 {'r2 mean': -0.39250254344957974, 'r2 std': 0.12580775916204226, 'rmse mean': 1.2256756765396788, 'rmse std': 0.03134051416727272}\n",
      "0.2 {'r2 mean': -0.31749276374875296, 'r2 std': 0.08553278328931085, 'rmse mean': 1.1375529639461315, 'rmse std': 0.015750861452665132}\n",
      "0.4 {'r2 mean': -0.26017779623073733, 'r2 std': 0.055308444560892074, 'rmse mean': 1.0445470539668247, 'rmse std': 0.010951785709823818}\n",
      "0.8 {'r2 mean': -0.24591894677668297, 'r2 std': 0.07278200661887449, 'rmse mean': 0.9747905023040854, 'rmse std': 0.02537517583569747}\n",
      "1.2126846264171962\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['exp'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -5.138898695745866, 'r2 std': 2.0745978979139648, 'rmse mean': 1.21798483418663, 'rmse std': 0.041654820314080944}\n",
      "0.025 {'r2 mean': -3.4746969803821144, 'r2 std': 1.581225315728574, 'rmse mean': 1.18531444029892, 'rmse std': 0.017163221313799902}\n",
      "0.05 {'r2 mean': -3.1177732878702273, 'r2 std': 0.8816238990343298, 'rmse mean': 1.147019893774949, 'rmse std': 0.01965770210179228}\n",
      "0.1 {'r2 mean': -2.3673709242595202, 'r2 std': 0.43302065966309283, 'rmse mean': 1.1139650795112228, 'rmse std': 0.009054802428297364}\n",
      "0.2 {'r2 mean': -1.9354788785403945, 'r2 std': 0.32491840889045187, 'rmse mean': 1.0850315828481132, 'rmse std': 0.014726596003720385}\n",
      "0.4 {'r2 mean': -1.5562850392618606, 'r2 std': 0.1757550304979939, 'rmse mean': 1.04027450608701, 'rmse std': 0.012022259501561602}\n",
      "0.8 {'r2 mean': -1.103239359346093, 'r2 std': 0.1469984539631951, 'rmse mean': 1.0030936386483205, 'rmse std': 0.023966281292409346}\n",
      "1.1132405679078807\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['exp'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4200 4200\n"
     ]
    }
   ],
   "source": [
    "x,X,y = extract_morgan(df['smiles'].values, df['exp'].values)\n",
    "print(len(X), len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -3.5699114095962132, 'r2 std': 0.562415140317653, 'rmse mean': 1.2262184108880299, 'rmse std': 0.03170948379212283}\n",
      "0.025 {'r2 mean': -1.7565779923596385, 'r2 std': 0.521640387818431, 'rmse mean': 1.2093114273384942, 'rmse std': 0.02610398016891242}\n",
      "0.05 {'r2 mean': -0.8158767188976908, 'r2 std': 0.19667718036370355, 'rmse mean': 1.1821426226362715, 'rmse std': 0.030748509534284395}\n",
      "0.1 {'r2 mean': -0.2786108567000312, 'r2 std': 0.07777330721442384, 'rmse mean': 1.2077505288339716, 'rmse std': 0.03141824341643375}\n",
      "0.2 {'r2 mean': -0.037994254388962846, 'r2 std': 0.0482498885240159, 'rmse mean': 1.256586849577109, 'rmse std': 0.022336417048210926}\n",
      "0.4 {'r2 mean': 0.0766426509251336, 'r2 std': 0.031666224533888405, 'rmse mean': 1.1582948604617265, 'rmse std': 0.01605699499162699}\n",
      "0.8 {'r2 mean': 0.12348131081271756, 'r2 std': 0.06561563129821961, 'rmse mean': 0.9774796738589574, 'rmse std': 0.0310760680529829}\n",
      "1.1739691962277943\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['exp'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -5.496100149814195, 'r2 std': 1.6477557320440832, 'rmse mean': 1.2523848794995998, 'rmse std': 0.05046478816706077}\n",
      "0.025 {'r2 mean': -4.037288582317525, 'r2 std': 1.499675123722066, 'rmse mean': 1.2320603445360159, 'rmse std': 0.042524814763589415}\n",
      "0.05 {'r2 mean': -2.9744137912117865, 'r2 std': 0.9670881190497821, 'rmse mean': 1.175391889813468, 'rmse std': 0.01733879831419753}\n",
      "0.1 {'r2 mean': -1.9483896244921113, 'r2 std': 0.3790627430086264, 'rmse mean': 1.1081855176669813, 'rmse std': 0.014830844985181117}\n",
      "0.2 {'r2 mean': -1.41154944486546, 'r2 std': 0.21151045164608523, 'rmse mean': 1.0541900466466454, 'rmse std': 0.016019474480455938}\n",
      "0.4 {'r2 mean': -0.7776631332809171, 'r2 std': 0.11066005958045351, 'rmse mean': 0.9836404529973091, 'rmse std': 0.015896163462819194}\n",
      "0.8 {'r2 mean': -0.2677760507394881, 'r2 std': 0.08952889721175714, 'rmse mean': 0.898774065367844, 'rmse std': 0.02372805875581283}\n",
      "1.100661028075409\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['exp'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (251)\n",
      "SMILES is too long (267)\n",
      "There are 4200 molecules. It will take a little time.\n",
      "(4200, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -0.9633031037797066, 'r2 std': 0.32109086740911513, 'rmse mean': 1.4704513844933393, 'rmse std': 0.09748913889357731}\n",
      "0.025 {'r2 mean': -0.8362444810353915, 'r2 std': 0.2711618051325478, 'rmse mean': 1.4049012245637262, 'rmse std': 0.08511606468713014}\n",
      "0.05 {'r2 mean': -0.9037585436110527, 'r2 std': 0.19961346378403874, 'rmse mean': 1.2701783005224592, 'rmse std': 0.038044560608227425}\n",
      "0.1 {'r2 mean': -0.9371134007588615, 'r2 std': 0.2291934273443875, 'rmse mean': 1.14604231808666, 'rmse std': 0.018055580856944975}\n",
      "0.2 {'r2 mean': -0.9232879200701618, 'r2 std': 0.11676848000115275, 'rmse mean': 1.0758186014492128, 'rmse std': 0.018429959946533816}\n",
      "0.4 {'r2 mean': -0.8128416321154386, 'r2 std': 0.08838500086006026, 'rmse mean': 1.016081269674474, 'rmse std': 0.014500728470382676}\n",
      "0.8 {'r2 mean': -0.7466333437024467, 'r2 std': 0.13221636398296832, 'rmse mean': 0.9760701193442218, 'rmse std': 0.021060696692366706}\n",
      "1.1942204597334418\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['exp'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'r2 mean': -4.807226717304896, 'r2 std': 1.542711331830811, 'rmse mean': 1.240416817101216, 'rmse std': 0.03783870471201192}\n",
      "0.025 {'r2 mean': -4.298655256057441, 'r2 std': 0.7887039691783326, 'rmse mean': 1.2183991870124626, 'rmse std': 0.01713413292401075}\n",
      "0.05 {'r2 mean': -3.8313331971983944, 'r2 std': 0.8248398735010598, 'rmse mean': 1.1799444176470322, 'rmse std': 0.014629011600167776}\n",
      "0.1 {'r2 mean': -2.8640841467999487, 'r2 std': 0.5293209430481294, 'rmse mean': 1.1405040142985226, 'rmse std': 0.012346620734383768}\n",
      "0.2 {'r2 mean': -2.5001389393180786, 'r2 std': 0.3000228112475364, 'rmse mean': 1.0934131303993178, 'rmse std': 0.0105391654536722}\n",
      "0.4 {'r2 mean': -1.7948884149636626, 'r2 std': 0.14025701942340474, 'rmse mean': 1.0445260298028665, 'rmse std': 0.014401397390565653}\n",
      "0.8 {'r2 mean': -1.1918861562251153, 'r2 std': 0.16766488537277058, 'rmse mean': 0.9885619223916103, 'rmse std': 0.023251177692112303}\n",
      "1.1293950740932897\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_regression(X, df['exp'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['rmse mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HIV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(41127, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>activity</th>\n",
       "      <th>HIV_active</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CCC1=[O+][Cu-3]2([O+]=C(CC)C1)[O+]=C(CC)CC(CC)...</td>\n",
       "      <td>CI</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C(=Cc1ccccc1)C1=[O+][Cu-3]2([O+]=C(C=Cc3ccccc3...</td>\n",
       "      <td>CI</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CC(=O)N1c2ccccc2Sc2c1ccc1ccccc21</td>\n",
       "      <td>CI</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Nc1ccc(C=Cc2ccc(N)cc2S(=O)(=O)O)c(S(=O)(=O)O)c1</td>\n",
       "      <td>CI</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=S(=O)(O)CCS(=O)(=O)O</td>\n",
       "      <td>CI</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles activity  HIV_active\n",
       "0  CCC1=[O+][Cu-3]2([O+]=C(CC)C1)[O+]=C(CC)CC(CC)...       CI           0\n",
       "1  C(=Cc1ccccc1)C1=[O+][Cu-3]2([O+]=C(C=Cc3ccccc3...       CI           0\n",
       "2                   CC(=O)N1c2ccccc2Sc2c1ccc1ccccc21       CI           0\n",
       "3    Nc1ccc(C=Cc2ccc(N)cc2S(=O)(=O)O)c(S(=O)(=O)O)c1       CI           0\n",
       "4                             O=S(=O)(O)CCS(=O)(=O)O       CI           0"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/hiv.csv')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (220)\n",
      "SMILES is too long (274)\n",
      "SMILES is too long (247)\n",
      "SMILES is too long (226)\n",
      "SMILES is too long (244)\n",
      "SMILES is too long (243)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (266)\n",
      "SMILES is too long (346)\n",
      "SMILES is too long (232)\n",
      "SMILES is too long (242)\n",
      "SMILES is too long (247)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (370)\n",
      "SMILES is too long (224)\n",
      "SMILES is too long (283)\n",
      "SMILES is too long (265)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (219)\n",
      "SMILES is too long (246)\n",
      "SMILES is too long (243)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (270)\n",
      "SMILES is too long (232)\n",
      "SMILES is too long (260)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (439)\n",
      "SMILES is too long (491)\n",
      "SMILES is too long (439)\n",
      "SMILES is too long (296)\n",
      "SMILES is too long (341)\n",
      "SMILES is too long (285)\n",
      "SMILES is too long (327)\n",
      "SMILES is too long (341)\n",
      "SMILES is too long (400)\n",
      "SMILES is too long (263)\n",
      "SMILES is too long (238)\n",
      "SMILES is too long (383)\n",
      "SMILES is too long (360)\n",
      "SMILES is too long (233)\n",
      "SMILES is too long (365)\n",
      "SMILES is too long (265)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (223)\n",
      "There are 41127 molecules. It will take a little time.\n",
      "(41127, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, xseg = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6058269475618208, 'roc_auc std': 0.02877319489109072, 'prc_auc mean': 0.09309772578417967, 'prc_auc std': 0.019935440405557676}\n",
      "0.025 {'roc_auc mean': 0.6343483087435325, 'roc_auc std': 0.017750497821389332, 'prc_auc mean': 0.11568207513414672, 'prc_auc std': 0.022407237161835535}\n",
      "0.05 {'roc_auc mean': 0.6703287141566399, 'roc_auc std': 0.021668911718422938, 'prc_auc mean': 0.14738642885614794, 'prc_auc std': 0.02483026398330456}\n",
      "0.1 {'roc_auc mean': 0.7003838553239822, 'roc_auc std': 0.009872218773290854, 'prc_auc mean': 0.18519380571901461, 'prc_auc std': 0.011136220300950446}\n",
      "0.2 {'roc_auc mean': 0.7291075118420648, 'roc_auc std': 0.005059223153495837, 'prc_auc mean': 0.2249364227344522, 'prc_auc std': 0.01287589617854118}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4 {'roc_auc mean': 0.7580561650674195, 'roc_auc std': 0.006143446737224518, 'prc_auc mean': 0.2638527670814344, 'prc_auc std': 0.010316790120602923}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8 {'roc_auc mean': 0.7718122559446297, 'roc_auc std': 0.0163079962594992, 'prc_auc mean': 0.3019600415211543, 'prc_auc std': 0.025510033450729528}\n",
      "0.6956948226628699\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['HIV_active'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6009842715009428, 'roc_auc std': 0.015687594435433685, 'prc_auc mean': 0.08916029878605873, 'prc_auc std': 0.02160966329844523}\n",
      "0.025 {'roc_auc mean': 0.6158154786059952, 'roc_auc std': 0.015050960869440966, 'prc_auc mean': 0.1112819107394398, 'prc_auc std': 0.016867872278305565}\n",
      "0.05 {'roc_auc mean': 0.6341460796491167, 'roc_auc std': 0.014440846343406354, 'prc_auc mean': 0.13271184912634376, 'prc_auc std': 0.015059606958925128}\n",
      "0.1 {'roc_auc mean': 0.6535084067311959, 'roc_auc std': 0.007941580440362686, 'prc_auc mean': 0.16729871812693847, 'prc_auc std': 0.012208350306345721}\n",
      "0.2 {'roc_auc mean': 0.6750337703282501, 'roc_auc std': 0.009385592369477256, 'prc_auc mean': 0.19723949552994988, 'prc_auc std': 0.011082762997165213}\n",
      "0.4 {'roc_auc mean': 0.6972244134258595, 'roc_auc std': 0.009199326848336731, 'prc_auc mean': 0.24295905393578415, 'prc_auc std': 0.014182094381008276}\n",
      "0.8 {'roc_auc mean': 0.7281668616130574, 'roc_auc std': 0.014179868762181348, 'prc_auc mean': 0.29109930248045657, 'prc_auc std': 0.029925359816465547}\n",
      "0.6578398974077739\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['HIV_active'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41127\n"
     ]
    }
   ],
   "source": [
    "x,X,_ = extract_morgan(df['smiles'].values,df['smiles'].values)\n",
    "print(len(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6434061125049021, 'roc_auc std': 0.02658233281050353, 'prc_auc mean': 0.14138119093633655, 'prc_auc std': 0.03959095575133971}\n",
      "0.025 {'roc_auc mean': 0.6700758260836535, 'roc_auc std': 0.024854197971553298, 'prc_auc mean': 0.19077185312919792, 'prc_auc std': 0.028653555661097993}\n",
      "0.05 {'roc_auc mean': 0.7069448949149395, 'roc_auc std': 0.01833419766695618, 'prc_auc mean': 0.2183756234132425, 'prc_auc std': 0.03410462523174302}\n",
      "0.1 {'roc_auc mean': 0.7322493580354331, 'roc_auc std': 0.008444744923267276, 'prc_auc mean': 0.2475998781386679, 'prc_auc std': 0.011745292045482773}\n",
      "0.2 {'roc_auc mean': 0.7581687981164059, 'roc_auc std': 0.008392829771356107, 'prc_auc mean': 0.27585535308445663, 'prc_auc std': 0.012042509601898658}\n",
      "0.4 {'roc_auc mean': 0.777382519316135, 'roc_auc std': 0.007651006862151206, 'prc_auc mean': 0.32516380349848156, 'prc_auc std': 0.013749622829105908}\n",
      "0.8 {'roc_auc mean': 0.7993075116194006, 'roc_auc std': 0.015177746021154532, 'prc_auc mean': 0.37277263562823737, 'prc_auc std': 0.020036476869293943}\n",
      "0.7267907172272672\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['HIV_active'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6024163644796384, 'roc_auc std': 0.01894606014411504, 'prc_auc mean': 0.12425504516762653, 'prc_auc std': 0.02529362590564165}\n",
      "0.025 {'roc_auc mean': 0.6371454913445063, 'roc_auc std': 0.017866339893140105, 'prc_auc mean': 0.1755562748488046, 'prc_auc std': 0.025912407564238817}\n",
      "0.05 {'roc_auc mean': 0.6700230987855677, 'roc_auc std': 0.01362091932698649, 'prc_auc mean': 0.22738234500910867, 'prc_auc std': 0.018665700539043085}\n",
      "0.1 {'roc_auc mean': 0.6997812429868648, 'roc_auc std': 0.012440072708295178, 'prc_auc mean': 0.2736317048561356, 'prc_auc std': 0.020412297840187412}\n",
      "0.2 {'roc_auc mean': 0.7333249099712664, 'roc_auc std': 0.0063630710394919025, 'prc_auc mean': 0.33295253431471467, 'prc_auc std': 0.010649021457543565}\n",
      "0.4 {'roc_auc mean': 0.7553342791476721, 'roc_auc std': 0.007581977731302513, 'prc_auc mean': 0.3881331284008115, 'prc_auc std': 0.014603312667043152}\n",
      "0.8 {'roc_auc mean': 0.7757824485470137, 'roc_auc std': 0.016260302485612353, 'prc_auc mean': 0.4254265840724215, 'prc_auc std': 0.026888118136493385}\n",
      "0.6962582621803612\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['HIV_active'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (220)\n",
      "SMILES is too long (274)\n",
      "SMILES is too long (247)\n",
      "SMILES is too long (226)\n",
      "SMILES is too long (244)\n",
      "SMILES is too long (243)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (266)\n",
      "SMILES is too long (346)\n",
      "SMILES is too long (232)\n",
      "SMILES is too long (242)\n",
      "SMILES is too long (247)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (370)\n",
      "SMILES is too long (224)\n",
      "SMILES is too long (283)\n",
      "SMILES is too long (265)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (219)\n",
      "SMILES is too long (246)\n",
      "SMILES is too long (243)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (270)\n",
      "SMILES is too long (232)\n",
      "SMILES is too long (260)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (439)\n",
      "SMILES is too long (491)\n",
      "SMILES is too long (439)\n",
      "SMILES is too long (296)\n",
      "SMILES is too long (341)\n",
      "SMILES is too long (285)\n",
      "SMILES is too long (327)\n",
      "SMILES is too long (341)\n",
      "SMILES is too long (400)\n",
      "SMILES is too long (263)\n",
      "SMILES is too long (238)\n",
      "SMILES is too long (383)\n",
      "SMILES is too long (360)\n",
      "SMILES is too long (233)\n",
      "SMILES is too long (365)\n",
      "SMILES is too long (265)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (223)\n",
      "There are 41127 molecules. It will take a little time.\n",
      "(41127, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6148152415078729, 'roc_auc std': 0.032461485517356295, 'prc_auc mean': 0.07947409350467056, 'prc_auc std': 0.024236189826321683}\n",
      "0.025 {'roc_auc mean': 0.6433598174337805, 'roc_auc std': 0.023476358442710926, 'prc_auc mean': 0.11498352496813777, 'prc_auc std': 0.017438886315830404}\n",
      "0.05 {'roc_auc mean': 0.6656978773069281, 'roc_auc std': 0.013947707865324458, 'prc_auc mean': 0.15014129785883662, 'prc_auc std': 0.017636156066503178}\n",
      "0.1 {'roc_auc mean': 0.6914223352844884, 'roc_auc std': 0.009746369715246998, 'prc_auc mean': 0.1840710730383871, 'prc_auc std': 0.012850900317852226}\n",
      "0.2 {'roc_auc mean': 0.7135218775772991, 'roc_auc std': 0.008346958344990572, 'prc_auc mean': 0.21496194949088904, 'prc_auc std': 0.009383557253301913}\n",
      "0.4 {'roc_auc mean': 0.7350730172743145, 'roc_auc std': 0.0056432581305330114, 'prc_auc mean': 0.24438771443099486, 'prc_auc std': 0.007513758617007545}\n",
      "0.8 {'roc_auc mean': 0.7552573946297683, 'roc_auc std': 0.01612827521108953, 'prc_auc mean': 0.2756497179733522, 'prc_auc std': 0.01968168423736812}\n",
      "0.6884496515734931\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['HIV_active'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.5919575769703936, 'roc_auc std': 0.024924714917260987, 'prc_auc mean': 0.08769632405860249, 'prc_auc std': 0.020178556083473353}\n",
      "0.025 {'roc_auc mean': 0.6070455576454541, 'roc_auc std': 0.015737270036636934, 'prc_auc mean': 0.1018562953354935, 'prc_auc std': 0.01922338471118357}\n",
      "0.05 {'roc_auc mean': 0.6226579395666584, 'roc_auc std': 0.012443627979488442, 'prc_auc mean': 0.12323771268239692, 'prc_auc std': 0.01854541682415088}\n",
      "0.1 {'roc_auc mean': 0.6463513839095539, 'roc_auc std': 0.009049910695534775, 'prc_auc mean': 0.1607213270117515, 'prc_auc std': 0.014620318475165775}\n",
      "0.2 {'roc_auc mean': 0.6716225720846729, 'roc_auc std': 0.005010180592615822, 'prc_auc mean': 0.19351015798058946, 'prc_auc std': 0.013187533710812762}\n",
      "0.4 {'roc_auc mean': 0.6983037974278389, 'roc_auc std': 0.008162377740575034, 'prc_auc mean': 0.24245930066321106, 'prc_auc std': 0.008282707739354241}\n",
      "0.8 {'roc_auc mean': 0.728421352755022, 'roc_auc std': 0.015724504751608543, 'prc_auc mean': 0.2977212414812652, 'prc_auc std': 0.028249643122643806}\n",
      "0.6523371686227992\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['HIV_active'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BACE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1513, 595)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mol</th>\n",
       "      <th>CID</th>\n",
       "      <th>Class</th>\n",
       "      <th>Model</th>\n",
       "      <th>pIC50</th>\n",
       "      <th>MW</th>\n",
       "      <th>AlogP</th>\n",
       "      <th>HBA</th>\n",
       "      <th>HBD</th>\n",
       "      <th>RB</th>\n",
       "      <th>...</th>\n",
       "      <th>PEOE6 (PEOE6)</th>\n",
       "      <th>PEOE7 (PEOE7)</th>\n",
       "      <th>PEOE8 (PEOE8)</th>\n",
       "      <th>PEOE9 (PEOE9)</th>\n",
       "      <th>PEOE10 (PEOE10)</th>\n",
       "      <th>PEOE11 (PEOE11)</th>\n",
       "      <th>PEOE12 (PEOE12)</th>\n",
       "      <th>PEOE13 (PEOE13)</th>\n",
       "      <th>PEOE14 (PEOE14)</th>\n",
       "      <th>canvasUID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>O1CC[C@@H](NC(=O)[C@@H](Cc2cc3cc(ccc3nc2N)-c2c...</td>\n",
       "      <td>BACE_1</td>\n",
       "      <td>1</td>\n",
       "      <td>Train</td>\n",
       "      <td>9.154901</td>\n",
       "      <td>431.56979</td>\n",
       "      <td>4.4014</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>53.205711</td>\n",
       "      <td>78.640335</td>\n",
       "      <td>226.85541</td>\n",
       "      <td>107.43491</td>\n",
       "      <td>37.133846</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.980170</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Fc1cc(cc(F)c1)C[C@H](NC(=O)[C@@H](N1CC[C@](NC(...</td>\n",
       "      <td>BACE_2</td>\n",
       "      <td>1</td>\n",
       "      <td>Train</td>\n",
       "      <td>8.853872</td>\n",
       "      <td>657.81073</td>\n",
       "      <td>2.6412</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>73.817162</td>\n",
       "      <td>47.171600</td>\n",
       "      <td>365.67694</td>\n",
       "      <td>174.07675</td>\n",
       "      <td>34.923889</td>\n",
       "      <td>7.980170</td>\n",
       "      <td>24.148668</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.663788</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S1(=O)(=O)N(c2cc(cc3c2n(cc3CC)CC1)C(=O)N[C@H](...</td>\n",
       "      <td>BACE_3</td>\n",
       "      <td>1</td>\n",
       "      <td>Train</td>\n",
       "      <td>8.698970</td>\n",
       "      <td>591.74091</td>\n",
       "      <td>2.5499</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>70.365707</td>\n",
       "      <td>47.941147</td>\n",
       "      <td>192.40652</td>\n",
       "      <td>255.75255</td>\n",
       "      <td>23.654478</td>\n",
       "      <td>0.230159</td>\n",
       "      <td>15.879790</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.663788</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>S1(=O)(=O)C[C@@H](Cc2cc(O[C@H](COCC)C(F)(F)F)c...</td>\n",
       "      <td>BACE_4</td>\n",
       "      <td>1</td>\n",
       "      <td>Train</td>\n",
       "      <td>8.698970</td>\n",
       "      <td>591.67828</td>\n",
       "      <td>3.1680</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>56.657166</td>\n",
       "      <td>37.954151</td>\n",
       "      <td>194.35304</td>\n",
       "      <td>202.76335</td>\n",
       "      <td>36.498634</td>\n",
       "      <td>0.980913</td>\n",
       "      <td>8.188327</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.385181</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S1(=O)(=O)N(c2cc(cc3c2n(cc3CC)CC1)C(=O)N[C@H](...</td>\n",
       "      <td>BACE_5</td>\n",
       "      <td>1</td>\n",
       "      <td>Train</td>\n",
       "      <td>8.698970</td>\n",
       "      <td>629.71283</td>\n",
       "      <td>3.5086</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>78.945702</td>\n",
       "      <td>39.361153</td>\n",
       "      <td>179.71288</td>\n",
       "      <td>220.46130</td>\n",
       "      <td>23.654478</td>\n",
       "      <td>0.230159</td>\n",
       "      <td>15.879790</td>\n",
       "      <td>0.0</td>\n",
       "      <td>26.100143</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 595 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 mol     CID  Class  Model  \\\n",
       "0  O1CC[C@@H](NC(=O)[C@@H](Cc2cc3cc(ccc3nc2N)-c2c...  BACE_1      1  Train   \n",
       "1  Fc1cc(cc(F)c1)C[C@H](NC(=O)[C@@H](N1CC[C@](NC(...  BACE_2      1  Train   \n",
       "2  S1(=O)(=O)N(c2cc(cc3c2n(cc3CC)CC1)C(=O)N[C@H](...  BACE_3      1  Train   \n",
       "3  S1(=O)(=O)C[C@@H](Cc2cc(O[C@H](COCC)C(F)(F)F)c...  BACE_4      1  Train   \n",
       "4  S1(=O)(=O)N(c2cc(cc3c2n(cc3CC)CC1)C(=O)N[C@H](...  BACE_5      1  Train   \n",
       "\n",
       "      pIC50         MW   AlogP  HBA  HBD  RB  ...  PEOE6 (PEOE6)  \\\n",
       "0  9.154901  431.56979  4.4014    3    2   5  ...      53.205711   \n",
       "1  8.853872  657.81073  2.6412    5    4  16  ...      73.817162   \n",
       "2  8.698970  591.74091  2.5499    4    3  11  ...      70.365707   \n",
       "3  8.698970  591.67828  3.1680    4    3  12  ...      56.657166   \n",
       "4  8.698970  629.71283  3.5086    3    3  11  ...      78.945702   \n",
       "\n",
       "   PEOE7 (PEOE7)  PEOE8 (PEOE8)  PEOE9 (PEOE9)  PEOE10 (PEOE10)  \\\n",
       "0      78.640335      226.85541      107.43491        37.133846   \n",
       "1      47.171600      365.67694      174.07675        34.923889   \n",
       "2      47.941147      192.40652      255.75255        23.654478   \n",
       "3      37.954151      194.35304      202.76335        36.498634   \n",
       "4      39.361153      179.71288      220.46130        23.654478   \n",
       "\n",
       "   PEOE11 (PEOE11)  PEOE12 (PEOE12)  PEOE13 (PEOE13)  PEOE14 (PEOE14)  \\\n",
       "0         0.000000         7.980170              0.0         0.000000   \n",
       "1         7.980170        24.148668              0.0        24.663788   \n",
       "2         0.230159        15.879790              0.0        24.663788   \n",
       "3         0.980913         8.188327              0.0        26.385181   \n",
       "4         0.230159        15.879790              0.0        26.100143   \n",
       "\n",
       "   canvasUID  \n",
       "0          1  \n",
       "1          2  \n",
       "2          3  \n",
       "3          4  \n",
       "4          5  \n",
       "\n",
       "[5 rows x 595 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/bace.csv')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1513 molecules. It will take a little time.\n",
      "(1513, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['mol'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.5472992953429163, 'roc_auc std': 0.06438200059142155, 'prc_auc mean': 0.4854464563974522, 'prc_auc std': 0.04565525660444758}\n",
      "0.025 {'roc_auc mean': 0.6333415348868185, 'roc_auc std': 0.02288325921102291, 'prc_auc mean': 0.5447058724642428, 'prc_auc std': 0.048897802767205956}\n",
      "0.05 {'roc_auc mean': 0.675508704642411, 'roc_auc std': 0.0006743101475881197, 'prc_auc mean': 0.6119071348410055, 'prc_auc std': 0.0004193213365963344}\n",
      "0.1 {'roc_auc mean': 0.7350123837664031, 'roc_auc std': 0.008215651342661001, 'prc_auc mean': 0.6770408903779559, 'prc_auc std': 0.002897029970534204}\n",
      "0.2 {'roc_auc mean': 0.7725600070354024, 'roc_auc std': 0.00024046785425724249, 'prc_auc mean': 0.7224426890682234, 'prc_auc std': 0.001557176017806361}\n",
      "0.4 {'roc_auc mean': 0.810305725946382, 'roc_auc std': 0.007338889024658568, 'prc_auc mean': 0.7533117262007103, 'prc_auc std': 0.0083568058160981}\n",
      "0.8 {'roc_auc mean': 0.8325208607817304, 'roc_auc std': 0.0007685551163811866, 'prc_auc mean': 0.7805360344523922, 'prc_auc std': 0.0025233014848645152}\n",
      "0.7152212160574376\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['Class'].values, rate, 20)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.5647000248829779, 'roc_auc std': 0.051608113286848195, 'prc_auc mean': 0.5143086315451196, 'prc_auc std': 0.04224550904840953}\n",
      "0.025 {'roc_auc mean': 0.5930792917557738, 'roc_auc std': 0.0347583755525642, 'prc_auc mean': 0.550936440105877, 'prc_auc std': 0.041474395352960367}\n",
      "0.05 {'roc_auc mean': 0.6407162986219518, 'roc_auc std': 0.02992161853303899, 'prc_auc mean': 0.5928644002687627, 'prc_auc std': 0.03115149180235131}\n",
      "0.1 {'roc_auc mean': 0.6718488202833057, 'roc_auc std': 0.018465218136911207, 'prc_auc mean': 0.627267261381173, 'prc_auc std': 0.023188344264376114}\n",
      "0.2 {'roc_auc mean': 0.7269076658403733, 'roc_auc std': 0.012233397900657571, 'prc_auc mean': 0.6822301659389838, 'prc_auc std': 0.013138867509688311}\n",
      "0.4 {'roc_auc mean': 0.7694387692758866, 'roc_auc std': 0.013246933588394764, 'prc_auc mean': 0.725595522799932, 'prc_auc std': 0.023337444800416537}\n",
      "0.8 {'roc_auc mean': 0.7985573122529643, 'roc_auc std': 0.018203779680689813, 'prc_auc mean': 0.7545404887359372, 'prc_auc std': 0.03148050916081607}\n",
      "0.6807497404161762\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['Class'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1513 1513\n"
     ]
    }
   ],
   "source": [
    "x,X,y = extract_morgan(df['mol'].values,df['Class'].values)\n",
    "print(len(X), len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6330038893176294, 'roc_auc std': 0.055543870121653675, 'prc_auc mean': 0.6181545968445523, 'prc_auc std': 0.039700393164874115}\n",
      "0.025 {'roc_auc mean': 0.7195311979694681, 'roc_auc std': 0.03771409738728807, 'prc_auc mean': 0.6853775480207699, 'prc_auc std': 0.03631031327176919}\n",
      "0.05 {'roc_auc mean': 0.7715284233420447, 'roc_auc std': 0.023877641848331854, 'prc_auc mean': 0.7334358933022341, 'prc_auc std': 0.026396847807101456}\n",
      "0.1 {'roc_auc mean': 0.8140372707916921, 'roc_auc std': 0.016099004338438132, 'prc_auc mean': 0.7818728465229869, 'prc_auc std': 0.019279184477085286}\n",
      "0.2 {'roc_auc mean': 0.8392861952214228, 'roc_auc std': 0.011598640628529128, 'prc_auc mean': 0.7953657730280368, 'prc_auc std': 0.016455445615877657}\n",
      "0.4 {'roc_auc mean': 0.8636919279552284, 'roc_auc std': 0.008441715073165556, 'prc_auc mean': 0.8264551952067171, 'prc_auc std': 0.014673562666545022}\n",
      "0.8 {'roc_auc mean': 0.8859804567413263, 'roc_auc std': 0.017765998289172424, 'prc_auc mean': 0.8501221898667073, 'prc_auc std': 0.025533810399274676}\n",
      "0.7895799087626874\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['Class'].values, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6218840381106248, 'roc_auc std': 0.05943011163864572, 'prc_auc mean': 0.585088468717016, 'prc_auc std': 0.06445068572470353}\n",
      "0.025 {'roc_auc mean': 0.66358417753835, 'roc_auc std': 0.050245634581102636, 'prc_auc mean': 0.6288548849242351, 'prc_auc std': 0.042989263925491183}\n",
      "0.05 {'roc_auc mean': 0.736159394446101, 'roc_auc std': 0.045961218745197076, 'prc_auc mean': 0.6976333689236892, 'prc_auc std': 0.0477176830385122}\n",
      "0.1 {'roc_auc mean': 0.790688493960198, 'roc_auc std': 0.01956506163627471, 'prc_auc mean': 0.7600174084014493, 'prc_auc std': 0.01812682222308056}\n",
      "0.2 {'roc_auc mean': 0.8296784738673277, 'roc_auc std': 0.013053890193062148, 'prc_auc mean': 0.7963438529992761, 'prc_auc std': 0.01784748328904429}\n",
      "0.4 {'roc_auc mean': 0.8584074635255016, 'roc_auc std': 0.011351048076646508, 'prc_auc mean': 0.8256306843599456, 'prc_auc std': 0.01528024365080733}\n",
      "0.8 {'roc_auc mean': 0.8776526130873957, 'roc_auc std': 0.01700912273277929, 'prc_auc mean': 0.8440485054824277, 'prc_auc std': 0.019416716700545896}\n",
      "0.7682935220764998\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['Class'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1513 molecules. It will take a little time.\n",
      "(1513, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['mol'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6014783734466169, 'roc_auc std': 0.036799114230400845, 'prc_auc mean': 0.5525628953302919, 'prc_auc std': 0.0392276221137332}\n",
      "0.025 {'roc_auc mean': 0.6393260173009613, 'roc_auc std': 0.0404591407367334, 'prc_auc mean': 0.5797846632319328, 'prc_auc std': 0.04635413692590368}\n",
      "0.05 {'roc_auc mean': 0.6842490114340393, 'roc_auc std': 0.03379057078194598, 'prc_auc mean': 0.6223747571271929, 'prc_auc std': 0.04232811140724257}\n",
      "0.1 {'roc_auc mean': 0.7361911227948206, 'roc_auc std': 0.023153299689448546, 'prc_auc mean': 0.673350796476712, 'prc_auc std': 0.027204234889600058}\n",
      "0.2 {'roc_auc mean': 0.7801361185465299, 'roc_auc std': 0.012367629704688793, 'prc_auc mean': 0.7221573875647064, 'prc_auc std': 0.013970183077379185}\n",
      "0.4 {'roc_auc mean': 0.8123199491678683, 'roc_auc std': 0.009957656875673276, 'prc_auc mean': 0.756535809559334, 'prc_auc std': 0.012027344398414604}\n",
      "0.8 {'roc_auc mean': 0.8351383399209485, 'roc_auc std': 0.020835566666908358, 'prc_auc mean': 0.7705092238940112, 'prc_auc std': 0.028837284832902713}\n",
      "0.7269769903731121\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['Class'].values, rate, 20, regularizer=True)\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.570048693102727, 'roc_auc std': 0.05316474841296403, 'prc_auc mean': 0.5388534168290537, 'prc_auc std': 0.04884817422902861}\n",
      "0.025 {'roc_auc mean': 0.6049137171907027, 'roc_auc std': 0.04636052506123286, 'prc_auc mean': 0.561890867213461, 'prc_auc std': 0.04983207113468062}\n",
      "0.05 {'roc_auc mean': 0.6821651299801019, 'roc_auc std': 0.02236570536345442, 'prc_auc mean': 0.6423885711164794, 'prc_auc std': 0.024554380763729074}\n",
      "0.1 {'roc_auc mean': 0.7316034804901366, 'roc_auc std': 0.024058119901495673, 'prc_auc mean': 0.6919387854187885, 'prc_auc std': 0.026221953854838023}\n",
      "0.2 {'roc_auc mean': 0.7725438613366166, 'roc_auc std': 0.014729453804810878, 'prc_auc mean': 0.7335557923836096, 'prc_auc std': 0.015761003346693708}\n",
      "0.4 {'roc_auc mean': 0.8121191866858917, 'roc_auc std': 0.010411745430853436, 'prc_auc mean': 0.7736674417098889, 'prc_auc std': 0.014850680953344145}\n",
      "0.8 {'roc_auc mean': 0.8456357048748353, 'roc_auc std': 0.02022410990259983, 'prc_auc mean': 0.8084463212044556, 'prc_auc std': 0.020162058876316374}\n",
      "0.7170042533801446\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['Class'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BBBP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2050, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>name</th>\n",
       "      <th>p_np</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Propanolol</td>\n",
       "      <td>1</td>\n",
       "      <td>[Cl].CC(C)NCC(O)COc1cccc2ccccc12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Terbutylchlorambucil</td>\n",
       "      <td>1</td>\n",
       "      <td>C(=O)(OC(C)(C)C)CCCc1ccc(cc1)N(CCCl)CCCl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>40730</td>\n",
       "      <td>1</td>\n",
       "      <td>c12c3c(N4CCN(C)CC4)c(F)cc1c(c(C(O)=O)cn2C(C)CO...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>C1CCN(CC1)Cc1cccc(c1)OCCCNC(=O)C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>cloxacillin</td>\n",
       "      <td>1</td>\n",
       "      <td>Cc1onc(c2ccccc2Cl)c1C(=O)N[C@H]3[C@H]4SC(C)(C)...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   num                  name  p_np  \\\n",
       "0    1            Propanolol     1   \n",
       "1    2  Terbutylchlorambucil     1   \n",
       "2    3                 40730     1   \n",
       "3    4                    24     1   \n",
       "4    5           cloxacillin     1   \n",
       "\n",
       "                                              smiles  \n",
       "0                   [Cl].CC(C)NCC(O)COc1cccc2ccccc12  \n",
       "1           C(=O)(OC(C)(C)C)CCCc1ccc(cc1)N(CCCl)CCCl  \n",
       "2  c12c3c(N4CCN(C)CC4)c(F)cc1c(c(C(O)=O)cn2C(C)CO...  \n",
       "3                   C1CCN(CC1)Cc1cccc(c1)OCCCNC(=O)C  \n",
       "4  Cc1onc(c2ccccc2Cl)c1C(=O)N[C@H]3[C@H]4SC(C)(C)...  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/bbbp.csv')\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (256)\n",
      "SMILES is too long (239)\n",
      "SMILES is too long (258)\n",
      "SMILES is too long (380)\n",
      "SMILES is too long (332)\n",
      "There are 2050 molecules. It will take a little time.\n",
      "(2050, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.8477537798146253, 'roc_auc std': 0.0382625905684465, 'prc_auc mean': 0.943581503243775, 'prc_auc std': 0.016433185725492738}\n",
      "0.025 {'roc_auc mean': 0.8679017157434888, 'roc_auc std': 0.02706057524079023, 'prc_auc mean': 0.9527760168022434, 'prc_auc std': 0.014111138153621016}\n",
      "0.05 {'roc_auc mean': 0.8809612539889471, 'roc_auc std': 0.013403111465178013, 'prc_auc mean': 0.957801425627426, 'prc_auc std': 0.006283944406237397}\n",
      "0.1 {'roc_auc mean': 0.8961332844216189, 'roc_auc std': 0.013010128716168537, 'prc_auc mean': 0.963239333004584, 'prc_auc std': 0.005588498078435249}\n",
      "0.2 {'roc_auc mean': 0.9132485889712505, 'roc_auc std': 0.006791484777813935, 'prc_auc mean': 0.9695546143714922, 'prc_auc std': 0.0039561474348626235}\n",
      "0.4 {'roc_auc mean': 0.9264326852531182, 'roc_auc std': 0.005838313473508262, 'prc_auc mean': 0.9748527401010156, 'prc_auc std': 0.003286753673322181}\n",
      "0.8 {'roc_auc mean': 0.9347979315569315, 'roc_auc std': 0.011786857580017258, 'prc_auc mean': 0.9774270081368103, 'prc_auc std': 0.005716715637666382}\n",
      "0.8953184628214258\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['p_np'].values, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.7738892139177351, 'roc_auc std': 0.05778231276676105, 'prc_auc mean': 0.9174420741937226, 'prc_auc std': 0.025648619622009705}\n",
      "0.025 {'roc_auc mean': 0.8441156098198108, 'roc_auc std': 0.022803476282383855, 'prc_auc mean': 0.9470147301780706, 'prc_auc std': 0.00843458527998391}\n",
      "0.05 {'roc_auc mean': 0.8666507547724708, 'roc_auc std': 0.015297212384543393, 'prc_auc mean': 0.9548286212918722, 'prc_auc std': 0.006067283038538704}\n",
      "0.1 {'roc_auc mean': 0.8927681584739544, 'roc_auc std': 0.0072929268570644445, 'prc_auc mean': 0.9637438048061812, 'prc_auc std': 0.002873548202867825}\n",
      "0.2 {'roc_auc mean': 0.9053243093603062, 'roc_auc std': 0.007057602653167909, 'prc_auc mean': 0.9681043986231168, 'prc_auc std': 0.0030236232077711024}\n",
      "0.4 {'roc_auc mean': 0.9207826485693325, 'roc_auc std': 0.0080481868222688, 'prc_auc mean': 0.9738185455992603, 'prc_auc std': 0.003349074216023665}\n",
      "0.8 {'roc_auc mean': 0.9348959191067487, 'roc_auc std': 0.010160331690105445, 'prc_auc mean': 0.9786988616539114, 'prc_auc std': 0.0037218227918384924}\n",
      "0.8769180877171942\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['p_np'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "O=N([O-])C1=C(CN=C1NCCSCc2ncccc2)Cc3ccccc3\n",
      "c1(nc(NC(N)=[NH2])sc1)CSCCNC(=[NH]C#N)NC\n",
      "Cc1nc(sc1)\\[NH]=C(\\N)N\n",
      "s1cc(CSCCN\\C(NC)=[NH]\\C#N)nc1\\[NH]=C(\\N)N\n",
      "c1c(c(ncc1)CSCCN\\C(=[NH]\\C#N)NCC)Br\n",
      "n1c(csc1\\[NH]=C(\\N)N)c1ccccc1\n",
      "n1c(csc1\\[NH]=C(\\N)N)c1cccc(c1)N\n",
      "n1c(csc1\\[NH]=C(\\N)N)c1cccc(c1)NC(C)=O\n",
      "n1c(csc1\\[NH]=C(\\N)N)c1cccc(c1)N\\C(NC)=[NH]\\C#N\n",
      "s1cc(nc1\\[NH]=C(\\N)N)C\n",
      "c1(cc(N\\C(=[NH]\\c2cccc(c2)CC)C)ccc1)CC\n",
      "2039 2039\n"
     ]
    }
   ],
   "source": [
    "x,X,y = extract_morgan(df['smiles'].values,df['p_np'].values)\n",
    "print(len(X), len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.733397288765292, 'roc_auc std': 0.03614102894062078, 'prc_auc mean': 0.8869217526658988, 'prc_auc std': 0.01708407192229991}\n",
      "0.025 {'roc_auc mean': 0.7643598387110389, 'roc_auc std': 0.03537169501979421, 'prc_auc mean': 0.8980970822004941, 'prc_auc std': 0.01693856951511407}\n",
      "0.05 {'roc_auc mean': 0.8072419657213994, 'roc_auc std': 0.02126581936334275, 'prc_auc mean': 0.9165218246524578, 'prc_auc std': 0.012710736196426887}\n",
      "0.1 {'roc_auc mean': 0.8336426914153131, 'roc_auc std': 0.016909855833264206, 'prc_auc mean': 0.9300688897337224, 'prc_auc std': 0.009872843134468035}\n",
      "0.2 {'roc_auc mean': 0.860766158618801, 'roc_auc std': 0.00908685211740163, 'prc_auc mean': 0.9417826275141049, 'prc_auc std': 0.006500961115931953}\n",
      "0.4 {'roc_auc mean': 0.8822577234686613, 'roc_auc std': 0.008220225217967354, 'prc_auc mean': 0.9509629264652538, 'prc_auc std': 0.005434902035362626}\n",
      "0.8 {'roc_auc mean': 0.8949803018162393, 'roc_auc std': 0.016497562742601345, 'prc_auc mean': 0.956723654122212, 'prc_auc std': 0.009009493124771803}\n",
      "0.8252351383595349\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, y, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6644190573925118, 'roc_auc std': 0.04468443905257435, 'prc_auc mean': 0.8656997871502143, 'prc_auc std': 0.01867608731621643}\n",
      "0.025 {'roc_auc mean': 0.6997999715802772, 'roc_auc std': 0.03134746167062131, 'prc_auc mean': 0.8754811898314149, 'prc_auc std': 0.015850062154400286}\n",
      "0.05 {'roc_auc mean': 0.7588367061124985, 'roc_auc std': 0.017724551446948858, 'prc_auc mean': 0.8999254445039998, 'prc_auc std': 0.008768701879477852}\n",
      "0.1 {'roc_auc mean': 0.7906409409549917, 'roc_auc std': 0.012951111603362455, 'prc_auc mean': 0.9124515891559837, 'prc_auc std': 0.006271077077412848}\n",
      "0.2 {'roc_auc mean': 0.8261535599236568, 'roc_auc std': 0.013928893581255262, 'prc_auc mean': 0.9279915440281139, 'prc_auc std': 0.006723918969329542}\n",
      "0.4 {'roc_auc mean': 0.8536683693910255, 'roc_auc std': 0.014392136377195242, 'prc_auc mean': 0.9401841991324094, 'prc_auc std': 0.006932595983987974}\n",
      "0.8 {'roc_auc mean': 0.8835361578525642, 'roc_auc std': 0.01714400885844468, 'prc_auc mean': 0.9532735096490862, 'prc_auc std': 0.008478850036079287}\n",
      "0.7824363947439323\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, y, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (256)\n",
      "SMILES is too long (239)\n",
      "SMILES is too long (258)\n",
      "SMILES is too long (380)\n",
      "SMILES is too long (332)\n",
      "There are 2050 molecules. It will take a little time.\n",
      "(2050, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.798261027957898, 'roc_auc std': 0.03347206706219177, 'prc_auc mean': 0.9218398475995772, 'prc_auc std': 0.013399290221888472}\n",
      "0.025 {'roc_auc mean': 0.8365727926545949, 'roc_auc std': 0.0214445303735446, 'prc_auc mean': 0.935230873426671, 'prc_auc std': 0.01500256842298215}\n",
      "0.05 {'roc_auc mean': 0.8542308080608558, 'roc_auc std': 0.0191224533714889, 'prc_auc mean': 0.9441560442247873, 'prc_auc std': 0.010888918668809009}\n",
      "0.1 {'roc_auc mean': 0.8971019809244314, 'roc_auc std': 0.011445053834116667, 'prc_auc mean': 0.9626201304336035, 'prc_auc std': 0.006063187452025117}\n",
      "0.2 {'roc_auc mean': 0.9201154233912618, 'roc_auc std': 0.006555199168391391, 'prc_auc mean': 0.9724782505703153, 'prc_auc std': 0.003174849020716298}\n",
      "0.4 {'roc_auc mean': 0.9337826485693326, 'roc_auc std': 0.006840293661709699, 'prc_auc mean': 0.9781057083380279, 'prc_auc std': 0.0034875629996734127}\n",
      "0.8 {'roc_auc mean': 0.9454365798227989, 'roc_auc std': 0.007897026831332312, 'prc_auc mean': 0.9824715192686524, 'prc_auc std': 0.003714577422357066}\n",
      "0.8836430373401676\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['p_np'].values, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.7391740001300116, 'roc_auc std': 0.031600047504738576, 'prc_auc mean': 0.9000188645846079, 'prc_auc std': 0.01421299778316027}\n",
      "0.025 {'roc_auc mean': 0.7854821464301197, 'roc_auc std': 0.02395510457882931, 'prc_auc mean': 0.9186413878134619, 'prc_auc std': 0.011022838930684169}\n",
      "0.05 {'roc_auc mean': 0.8215325239117363, 'roc_auc std': 0.019433142823767825, 'prc_auc mean': 0.935148571302029, 'prc_auc std': 0.009093281566665177}\n",
      "0.1 {'roc_auc mean': 0.8615004075976197, 'roc_auc std': 0.013845278402217385, 'prc_auc mean': 0.9516913126752362, 'prc_auc std': 0.0058654148611723585}\n",
      "0.2 {'roc_auc mean': 0.8880028984968309, 'roc_auc std': 0.010491720876010052, 'prc_auc mean': 0.9615230841399451, 'prc_auc std': 0.00474501035599686}\n",
      "0.4 {'roc_auc mean': 0.9119402054292005, 'roc_auc std': 0.0069590467820690935, 'prc_auc mean': 0.9705769841506386, 'prc_auc std': 0.0026481028860411097}\n",
      "0.8 {'roc_auc mean': 0.9362035176706959, 'roc_auc std': 0.012307570606022555, 'prc_auc mean': 0.979157664590286, 'prc_auc std': 0.0046031443290584576}\n",
      "0.849119385666602\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification(X, df['p_np'].values, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tox21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7831, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NR-AR</th>\n",
       "      <th>NR-AR-LBD</th>\n",
       "      <th>NR-AhR</th>\n",
       "      <th>NR-Aromatase</th>\n",
       "      <th>NR-ER</th>\n",
       "      <th>NR-ER-LBD</th>\n",
       "      <th>NR-PPAR-gamma</th>\n",
       "      <th>SR-ARE</th>\n",
       "      <th>SR-ATAD5</th>\n",
       "      <th>SR-HSE</th>\n",
       "      <th>SR-MMP</th>\n",
       "      <th>SR-p53</th>\n",
       "      <th>mol_id</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TOX3021</td>\n",
       "      <td>CCOc1ccc2nc(S(N)(=O)=O)sc2c1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TOX3020</td>\n",
       "      <td>CCN1C(=O)NC(c2ccccc2)C1=O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TOX3024</td>\n",
       "      <td>CC[C@]1(O)CC[C@H]2[C@@H]3CCC4=CCCC[C@@H]4[C@H]...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TOX3027</td>\n",
       "      <td>CCCN(CC)C(CC)C(=O)Nc1c(C)cccc1C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>TOX20800</td>\n",
       "      <td>CC(O)(P(=O)(O)O)P(=O)(O)O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   NR-AR  NR-AR-LBD  NR-AhR  NR-Aromatase  NR-ER  NR-ER-LBD  NR-PPAR-gamma  \\\n",
       "0    0.0        0.0     1.0           NaN    NaN        0.0            0.0   \n",
       "1    0.0        0.0     0.0           0.0    0.0        0.0            0.0   \n",
       "2    NaN        NaN     NaN           NaN    NaN        NaN            NaN   \n",
       "3    0.0        0.0     0.0           0.0    0.0        0.0            0.0   \n",
       "4    0.0        0.0     0.0           0.0    0.0        0.0            0.0   \n",
       "\n",
       "   SR-ARE  SR-ATAD5  SR-HSE  SR-MMP  SR-p53    mol_id  \\\n",
       "0     1.0       0.0     0.0     0.0     0.0   TOX3021   \n",
       "1     NaN       0.0     NaN     0.0     0.0   TOX3020   \n",
       "2     0.0       NaN     0.0     NaN     NaN   TOX3024   \n",
       "3     NaN       0.0     NaN     0.0     0.0   TOX3027   \n",
       "4     0.0       0.0     0.0     0.0     0.0  TOX20800   \n",
       "\n",
       "                                              smiles  \n",
       "0                       CCOc1ccc2nc(S(N)(=O)=O)sc2c1  \n",
       "1                          CCN1C(=O)NC(c2ccccc2)C1=O  \n",
       "2  CC[C@]1(O)CC[C@H]2[C@@H]3CCC4=CCCC[C@@H]4[C@H]...  \n",
       "3                    CCCN(CC)C(CC)C(=O)Nc1c(C)cccc1C  \n",
       "4                          CC(O)(P(=O)(O)O)P(=O)(O)O  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/tox21.csv')\n",
    "KEYS  = df.keys()[:-2]\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (231)\n",
      "SMILES is too long (263)\n",
      "SMILES is too long (277)\n",
      "SMILES is too long (219)\n",
      "SMILES is too long (325)\n",
      "SMILES is too long (243)\n",
      "SMILES is too long (271)\n",
      "SMILES is too long (255)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (235)\n",
      "SMILES is too long (227)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (248)\n",
      "SMILES is too long (264)\n",
      "SMILES is too long (311)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (340)\n",
      "SMILES is too long (230)\n",
      "SMILES is too long (306)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (233)\n",
      "SMILES is too long (221)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (225)\n",
      "SMILES is too long (264)\n",
      "SMILES is too long (271)\n",
      "SMILES is too long (226)\n",
      "SMILES is too long (275)\n",
      "SMILES is too long (225)\n",
      "SMILES is too long (273)\n",
      "There are 7831 molecules. It will take a little time.\n",
      "(7831, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.64151909310844, 'roc_auc std': 0.016050028861588773, 'prc_auc mean': 0.1758582910211457, 'prc_auc std': 0.01529279413384056}\n",
      "0.025 {'roc_auc mean': 0.6665358182643287, 'roc_auc std': 0.0117894605608774, 'prc_auc mean': 0.19084844176679733, 'prc_auc std': 0.010539737554377378}\n",
      "0.05 {'roc_auc mean': 0.6881717960894373, 'roc_auc std': 0.010121960060572349, 'prc_auc mean': 0.21599316046408723, 'prc_auc std': 0.008952086479475296}\n",
      "0.1 {'roc_auc mean': 0.7105856907029156, 'roc_auc std': 0.007645973769303752, 'prc_auc mean': 0.24694069228845303, 'prc_auc std': 0.004384371768915379}\n",
      "0.2 {'roc_auc mean': 0.7341224095833864, 'roc_auc std': 0.005245935473006969, 'prc_auc mean': 0.2791287121274567, 'prc_auc std': 0.0063296584319239985}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4 {'roc_auc mean': 0.757260316106741, 'roc_auc std': 0.003689677713792711, 'prc_auc mean': 0.31056628944501397, 'prc_auc std': 0.006641182785945235}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8 {'roc_auc mean': 0.7803482764439821, 'roc_auc std': 0.006867558410256675, 'prc_auc mean': 0.3510005964574861, 'prc_auc std': 0.010298133766192964}\n",
      "0.711220485757033\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6168919698982802, 'roc_auc std': 0.010209742163878866, 'prc_auc mean': 0.14505578352784826, 'prc_auc std': 0.008860706728140077}\n",
      "0.025 {'roc_auc mean': 0.6437860173673717, 'roc_auc std': 0.008322097977464188, 'prc_auc mean': 0.17249189093881948, 'prc_auc std': 0.011852962232863263}\n",
      "0.05 {'roc_auc mean': 0.659035480603624, 'roc_auc std': 0.005610489466649485, 'prc_auc mean': 0.19145609146312456, 'prc_auc std': 0.008087036355671106}\n",
      "0.1 {'roc_auc mean': 0.6794884073242539, 'roc_auc std': 0.005523386417234203, 'prc_auc mean': 0.22210550038749405, 'prc_auc std': 0.009045918567076282}\n",
      "0.2 {'roc_auc mean': 0.7004202498663027, 'roc_auc std': 0.004955937078233858, 'prc_auc mean': 0.2535485842143325, 'prc_auc std': 0.005401556881354955}\n",
      "0.4 {'roc_auc mean': 0.7190726385467954, 'roc_auc std': 0.004263675954611791, 'prc_auc mean': 0.2833259095447986, 'prc_auc std': 0.0037698661086908313}\n",
      "0.8 {'roc_auc mean': 0.7401952203765176, 'roc_auc std': 0.00621103443446874, 'prc_auc mean': 0.32340261626969163, 'prc_auc std': 0.010137238199987622}\n",
      "0.6798414262833065\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7831\n"
     ]
    }
   ],
   "source": [
    "x,X,_ = extract_morgan(df['smiles'].values,df['smiles'].values)\n",
    "print(len(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6333624508315959, 'roc_auc std': 0.01567442937330042, 'prc_auc mean': 0.18883913379179415, 'prc_auc std': 0.01824341166311773}\n",
      "0.025 {'roc_auc mean': 0.6644288369817, 'roc_auc std': 0.012346058070839834, 'prc_auc mean': 0.22319987415179116, 'prc_auc std': 0.01555880611736237}\n",
      "0.05 {'roc_auc mean': 0.6860526707675982, 'roc_auc std': 0.008946215596877062, 'prc_auc mean': 0.2540264780612906, 'prc_auc std': 0.014641450236352969}\n",
      "0.1 {'roc_auc mean': 0.7132230781526047, 'roc_auc std': 0.005734801673953929, 'prc_auc mean': 0.286615693869996, 'prc_auc std': 0.006779465105855}\n",
      "0.2 {'roc_auc mean': 0.7396379974255787, 'roc_auc std': 0.004642685912838342, 'prc_auc mean': 0.3241177598411691, 'prc_auc std': 0.007039027697871492}\n",
      "0.4 {'roc_auc mean': 0.7587283799738419, 'roc_auc std': 0.004465444053088761, 'prc_auc mean': 0.35030560070416666, 'prc_auc std': 0.005718710446595417}\n",
      "0.8 {'roc_auc mean': 0.7755816160410125, 'roc_auc std': 0.0071868010399591, 'prc_auc mean': 0.38101896586719464, 'prc_auc std': 0.013550301838057877}\n",
      "0.7101450043105617\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.5921641779111064, 'roc_auc std': 0.010128830045598829, 'prc_auc mean': 0.15476197688942223, 'prc_auc std': 0.016767739981842538}\n",
      "0.025 {'roc_auc mean': 0.6164608192173688, 'roc_auc std': 0.007696774834248351, 'prc_auc mean': 0.172398000211232, 'prc_auc std': 0.014272201941834298}\n",
      "0.05 {'roc_auc mean': 0.6397774811750119, 'roc_auc std': 0.007570479308828419, 'prc_auc mean': 0.20492211653298847, 'prc_auc std': 0.010240024729650448}\n",
      "0.1 {'roc_auc mean': 0.6683661833964416, 'roc_auc std': 0.004972631200908763, 'prc_auc mean': 0.24535057387582765, 'prc_auc std': 0.00801940445357613}\n",
      "0.2 {'roc_auc mean': 0.6921022105860207, 'roc_auc std': 0.0055107716993896255, 'prc_auc mean': 0.2944970071194509, 'prc_auc std': 0.005891127860687563}\n",
      "0.4 {'roc_auc mean': 0.7234508880052891, 'roc_auc std': 0.0034730041474466265, 'prc_auc mean': 0.34455464724966756, 'prc_auc std': 0.005509459429821029}\n",
      "0.8 {'roc_auc mean': 0.7518463883868656, 'roc_auc std': 0.008565150963933007, 'prc_auc mean': 0.40187654405134887, 'prc_auc std': 0.014209916759108022}\n",
      "0.6691668783825863\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (231)\n",
      "SMILES is too long (263)\n",
      "SMILES is too long (277)\n",
      "SMILES is too long (219)\n",
      "SMILES is too long (325)\n",
      "SMILES is too long (243)\n",
      "SMILES is too long (271)\n",
      "SMILES is too long (255)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (235)\n",
      "SMILES is too long (227)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (248)\n",
      "SMILES is too long (264)\n",
      "SMILES is too long (311)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (340)\n",
      "SMILES is too long (230)\n",
      "SMILES is too long (306)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (233)\n",
      "SMILES is too long (221)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (225)\n",
      "SMILES is too long (264)\n",
      "SMILES is too long (271)\n",
      "SMILES is too long (226)\n",
      "SMILES is too long (275)\n",
      "SMILES is too long (225)\n",
      "SMILES is too long (273)\n",
      "There are 7831 molecules. It will take a little time.\n",
      "(7831, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6287069922634315, 'roc_auc std': 0.014101236740853996, 'prc_auc mean': 0.1526099594865217, 'prc_auc std': 0.01482841886038846}\n",
      "0.025 {'roc_auc mean': 0.6559610266525667, 'roc_auc std': 0.011568654293008051, 'prc_auc mean': 0.1753292371030141, 'prc_auc std': 0.0147876262395851}\n",
      "0.05 {'roc_auc mean': 0.6806685358152487, 'roc_auc std': 0.007098046151958889, 'prc_auc mean': 0.199577586467581, 'prc_auc std': 0.011721314207633223}\n",
      "0.1 {'roc_auc mean': 0.7114063401697189, 'roc_auc std': 0.004624559267991757, 'prc_auc mean': 0.23639928686555417, 'prc_auc std': 0.004900666217932707}\n",
      "0.2 {'roc_auc mean': 0.739379565861692, 'roc_auc std': 0.004337354139378147, 'prc_auc mean': 0.27595774517451926, 'prc_auc std': 0.006859191695338786}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4 {'roc_auc mean': 0.7642522060505831, 'roc_auc std': 0.004134499997318782, 'prc_auc mean': 0.3126934743892206, 'prc_auc std': 0.005675516569271745}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/home/honda/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8 {'roc_auc mean': 0.7820143548965414, 'roc_auc std': 0.006381915143904057, 'prc_auc mean': 0.3460685729246834, 'prc_auc std': 0.008372182220558747}\n",
      "0.7089127173871118\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.598776054830814, 'roc_auc std': 0.013924447399479, 'prc_auc mean': 0.1364523786353374, 'prc_auc std': 0.013000723641505502}\n",
      "0.025 {'roc_auc mean': 0.6221811521899527, 'roc_auc std': 0.01156029423686949, 'prc_auc mean': 0.14888363423989687, 'prc_auc std': 0.010114044404021192}\n",
      "0.05 {'roc_auc mean': 0.6435134317340592, 'roc_auc std': 0.005750276984740776, 'prc_auc mean': 0.17977695401778637, 'prc_auc std': 0.009800834094743884}\n",
      "0.1 {'roc_auc mean': 0.6638720335878109, 'roc_auc std': 0.004754730130406659, 'prc_auc mean': 0.2157620937433676, 'prc_auc std': 0.005955836557750569}\n",
      "0.2 {'roc_auc mean': 0.686634084685952, 'roc_auc std': 0.003742938833910734, 'prc_auc mean': 0.24968296188999634, 'prc_auc std': 0.005773174673727184}\n",
      "0.4 {'roc_auc mean': 0.7091303709027542, 'roc_auc std': 0.0033690762782451265, 'prc_auc mean': 0.2895359559285539, 'prc_auc std': 0.004537896364585201}\n",
      "0.8 {'roc_auc mean': 0.7330542842247323, 'roc_auc std': 0.009401925943568597, 'prc_auc mean': 0.34028475525496776, 'prc_auc std': 0.014915161949269598}\n",
      "0.6653087731651536\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ClinTox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1484, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>FDA_APPROVED</th>\n",
       "      <th>CT_TOX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>*C(=O)[C@H](CCCCNC(=O)OCCOC)NC(=O)OCCOC</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[C@@H]1([C@@H]([C@@H]([C@H]([C@@H]([C@@H]1Cl)C...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[C@H]([C@@H]([C@@H](C(=O)[O-])O)O)([C@H](C(=O)...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[H]/[NH+]=C(/C1=CC(=O)/C(=C\\C=c2ccc(=C([NH3+])...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[H]/[NH+]=C(\\N)/c1ccc(cc1)OCCCCCOc2ccc(cc2)/C(...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles  FDA_APPROVED  CT_TOX\n",
       "0            *C(=O)[C@H](CCCCNC(=O)OCCOC)NC(=O)OCCOC             1       0\n",
       "1  [C@@H]1([C@@H]([C@@H]([C@H]([C@@H]([C@@H]1Cl)C...             1       0\n",
       "2  [C@H]([C@@H]([C@@H](C(=O)[O-])O)O)([C@H](C(=O)...             1       0\n",
       "3  [H]/[NH+]=C(/C1=CC(=O)/C(=C\\C=c2ccc(=C([NH3+])...             1       0\n",
       "4  [H]/[NH+]=C(\\N)/c1ccc(cc1)OCCCCCOc2ccc(cc2)/C(...             1       0"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/clintox.csv')\n",
    "KEYS  = df.keys()[1:]\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (219)\n",
      "SMILES is too long (263)\n",
      "SMILES is too long (318)\n",
      "SMILES is too long (238)\n",
      "SMILES is too long (230)\n",
      "SMILES is too long (227)\n",
      "SMILES is too long (261)\n",
      "SMILES is too long (227)\n",
      "SMILES is too long (279)\n",
      "SMILES is too long (255)\n",
      "SMILES is too long (271)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (221)\n",
      "SMILES is too long (225)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (314)\n",
      "SMILES is too long (236)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (339)\n",
      "There are 1484 molecules. It will take a little time.\n",
      "(1484, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = trfm.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.9018242661384992, 'roc_auc std': 0.04313171534934718, 'prc_auc mean': 0.7965750004752625, 'prc_auc std': 0.1255654538042044}\n",
      "0.025 {'roc_auc mean': 0.9538569624756252, 'roc_auc std': 0.011884728005462383, 'prc_auc mean': 0.8900095517695427, 'prc_auc std': 0.0348635138699111}\n",
      "0.05 {'roc_auc mean': 0.9566091006545839, 'roc_auc std': 0.008181585471669514, 'prc_auc mean': 0.9064266675913085, 'prc_auc std': 0.020830167471257887}\n",
      "0.1 {'roc_auc mean': 0.9622426657882839, 'roc_auc std': 0.007016233718590539, 'prc_auc mean': 0.9120507248499441, 'prc_auc std': 0.027366924749408095}\n",
      "0.2 {'roc_auc mean': 0.971071167130632, 'roc_auc std': 0.00647415346593895, 'prc_auc mean': 0.9384134813089668, 'prc_auc std': 0.012469480124946988}\n",
      "0.4 {'roc_auc mean': 0.9775454143775297, 'roc_auc std': 0.0051896771390132585, 'prc_auc mean': 0.948108042625295, 'prc_auc std': 0.012717029098288977}\n",
      "0.8 {'roc_auc mean': 0.9818991194169502, 'roc_auc std': 0.00982588802347242, 'prc_auc mean': 0.961145662320385, 'prc_auc std': 0.01719425763618896}\n",
      "0.9578640994260149\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.744048726732585, 'roc_auc std': 0.08683135390874176, 'prc_auc mean': 0.6684055334820516, 'prc_auc std': 0.09308369877997541}\n",
      "0.025 {'roc_auc mean': 0.8620829750873362, 'roc_auc std': 0.0392416138404536, 'prc_auc mean': 0.7934747646715279, 'prc_auc std': 0.07690297797496883}\n",
      "0.05 {'roc_auc mean': 0.895731056554226, 'roc_auc std': 0.02381434997565653, 'prc_auc mean': 0.8365750109775829, 'prc_auc std': 0.04016195241074952}\n",
      "0.1 {'roc_auc mean': 0.919257584012936, 'roc_auc std': 0.01848738830377069, 'prc_auc mean': 0.8628205515905792, 'prc_auc std': 0.05192267045138033}\n",
      "0.2 {'roc_auc mean': 0.9329027133381448, 'roc_auc std': 0.014560155381545606, 'prc_auc mean': 0.8965803701363869, 'prc_auc std': 0.01717608461207941}\n",
      "0.4 {'roc_auc mean': 0.9515344277288882, 'roc_auc std': 0.01036903086033248, 'prc_auc mean': 0.913172604608555, 'prc_auc std': 0.014585218370415794}\n",
      "0.8 {'roc_auc mean': 0.9568551810139537, 'roc_auc std': 0.016972417711099527, 'prc_auc mean': 0.9322028416834245, 'prc_auc std': 0.023623015674209512}\n",
      "0.8946303806382957\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ECFP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_ecfp_multi(smiles, targets):\n",
    "    x,X,y = [],[],[]\n",
    "    for sm,target in zip(smiles,targets):\n",
    "        mol = Chem.MolFromSmiles(sm)\n",
    "        if mol is None:\n",
    "            print(sm)\n",
    "            continue\n",
    "        fp = AllChem.GetMorganFingerprintAsBitVect(mol, 2, 1024) # Morgan (Similar to ECFP4)\n",
    "        x.append(sm)\n",
    "        X.append(bit2np(fp))\n",
    "        y.append(target)\n",
    "    return x,np.array(X),np.array(y)\n",
    "\n",
    "def evaluate_mlp_classification_multi(X, y, rate, n_repeats, model='ridge'):\n",
    "    auc = np.empty(n_repeats)\n",
    "    for i in range(n_repeats):\n",
    "        _auc = np.empty(len(KEYS))\n",
    "        for j,key in enumerate(KEYS):\n",
    "            X_dr = X\n",
    "            y_dr = y[:,j]\n",
    "            if model=='ridge':\n",
    "                clf = LogisticRegression(penalty='l2', solver='lbfgs', max_iter=1000)\n",
    "            elif model=='rf':\n",
    "                clf = RandomForestClassifier(n_estimators=10)\n",
    "            else:\n",
    "                raise ValueError('Model \"{}\" is invalid. Specify \"ridge\" or \"rf\".'.format(model))\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X_dr, y_dr, test_size=1-rate, stratify=y_dr)\n",
    "            clf.fit(X_train, y_train)\n",
    "            y_score = clf.predict_proba(X_test)\n",
    "            _auc[j] = roc_auc_score(y_test, y_score[:,1])\n",
    "        auc[i] = np.mean(_auc)\n",
    "    ret = {}\n",
    "    ret['roc_auc mean'] = np.mean(auc)\n",
    "    ret['roc_auc std'] = np.std(auc)\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NH4][Pt]([NH4])(Cl)Cl\n",
      "c1ccc(cc1)n2c(=O)c(c(=O)n2c3ccccc3)CCS(=O)c4ccccc4\n",
      "Cc1cc2c(cc1C)N3C=N2[Co+]456(N7=C8[C@H](C(C7=CC9=N4C(=C(C1=N5[C@@]([C@@H]2N6C(=C8C)[C@@]([C@H]2CC(=O)N)(CCC(=O)NC[C@H](OP(=O)(O[C@@H]2[C@H](O[C@H]3[C@@H]2O)CO)[O-])C)C)([C@@]([C@@H]1CCC(=O)N)(C)CC(=O)N)C)C)[C@@]([C@@H]9CCC(=O)N)(C)CC(=O)N)(C)C)CCC(=O)N)O\n",
      "Cc1cc2c(cc1C)N3C=N2[Co]456(N7=C8[C@H](C(C7=CC9=N4C(=C(C1=N5[C@@]([C@@H]2N6C(=C8C)[C@@]([C@H]2CC(=O)N)(CCC(=O)NC[C@H](OP(=O)(O[C@@H]2[C@H](O[C@H]3[C@@H]2O)CO)O)C)C)([C@@]([C@@H]1CCC(=O)N)(C)CC(=O)N)C)C)[C@@]([C@@H]9CCC(=O)N)(C)CC(=O)N)(C)C)CCC(=O)N)C#N\n",
      "CCCCc1c(=O)n(n(c1=O)c2ccc(cc2)O)c3ccccc3\n",
      "CCCCc1c(=O)n(n(c1=O)c2ccccc2)c3ccccc3\n",
      "1478\n"
     ]
    }
   ],
   "source": [
    "x,X,y = extract_ecfp_multi(df['smiles'].values, np.array([df['FDA_APPROVED'].values, df['CT_TOX'].values]).T)\n",
    "print(len(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.5961327148497889, 'roc_auc std': 0.04730622642300294}\n",
      "0.025 {'roc_auc mean': 0.6315359454621499, 'roc_auc std': 0.03746712260611319}\n",
      "0.05 {'roc_auc mean': 0.6670765122873054, 'roc_auc std': 0.041667293874089344}\n",
      "0.1 {'roc_auc mean': 0.7111053771242182, 'roc_auc std': 0.029331759695053892}\n",
      "0.2 {'roc_auc mean': 0.7427732149561315, 'roc_auc std': 0.0210930976648566}\n",
      "0.4 {'roc_auc mean': 0.7752242414154089, 'roc_auc std': 0.0182310753220062}\n",
      "0.8 {'roc_auc mean': 0.8059353345796127, 'roc_auc std': 0.023527546343130177}\n",
      "0.7042547629535166\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_mlp_classification_multi(X,y, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.5559137633063427, 'roc_auc std': 0.028615916176213715}\n",
      "0.025 {'roc_auc mean': 0.5661714134905885, 'roc_auc std': 0.028829411151434555}\n",
      "0.05 {'roc_auc mean': 0.579895213322555, 'roc_auc std': 0.022711743369155733}\n",
      "0.1 {'roc_auc mean': 0.5926863338993428, 'roc_auc std': 0.019354589748809537}\n",
      "0.2 {'roc_auc mean': 0.628754556399272, 'roc_auc std': 0.020596421076075414}\n",
      "0.4 {'roc_auc mean': 0.6513588822599476, 'roc_auc std': 0.020985655222913676}\n",
      "0.8 {'roc_auc mean': 0.6896738399124435, 'roc_auc std': 0.03571882329052343}\n",
      "0.6092077146557846\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_mlp_classification_multi(X,y, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SMILES is too long (219)\n",
      "SMILES is too long (263)\n",
      "SMILES is too long (318)\n",
      "SMILES is too long (238)\n",
      "SMILES is too long (230)\n",
      "SMILES is too long (227)\n",
      "SMILES is too long (261)\n",
      "SMILES is too long (227)\n",
      "SMILES is too long (279)\n",
      "SMILES is too long (255)\n",
      "SMILES is too long (271)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (253)\n",
      "SMILES is too long (251)\n",
      "SMILES is too long (221)\n",
      "SMILES is too long (225)\n",
      "SMILES is too long (284)\n",
      "SMILES is too long (314)\n",
      "SMILES is too long (236)\n",
      "SMILES is too long (240)\n",
      "SMILES is too long (339)\n",
      "There are 1484 molecules. It will take a little time.\n",
      "(1484, 1024)\n"
     ]
    }
   ],
   "source": [
    "x_split = [split(sm) for sm in df['smiles'].values]\n",
    "xid, _ = get_array(x_split)\n",
    "X = rnn.encode(torch.t(xid))\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.7901042845064358, 'roc_auc std': 0.0664864925578943, 'prc_auc mean': 0.6632765625444268, 'prc_auc std': 0.0960471342296363}\n",
      "0.025 {'roc_auc mean': 0.8716745864545367, 'roc_auc std': 0.035690992621761536, 'prc_auc mean': 0.7322497110443599, 'prc_auc std': 0.08143064059408597}\n",
      "0.05 {'roc_auc mean': 0.9086472512207727, 'roc_auc std': 0.014959624295857515, 'prc_auc mean': 0.7826022613130412, 'prc_auc std': 0.06903059884536064}\n",
      "0.1 {'roc_auc mean': 0.9360556671298463, 'roc_auc std': 0.011053184882245267, 'prc_auc mean': 0.8578831182376747, 'prc_auc std': 0.04199820917088663}\n",
      "0.2 {'roc_auc mean': 0.9576795477742481, 'roc_auc std': 0.008187961511857828, 'prc_auc mean': 0.9112466832143771, 'prc_auc std': 0.022791284402359738}\n",
      "0.4 {'roc_auc mean': 0.9670277680866969, 'roc_auc std': 0.005826937078495892, 'prc_auc mean': 0.9388679093563305, 'prc_auc std': 0.011330079622427578}\n",
      "0.8 {'roc_auc mean': 0.9761307465554309, 'roc_auc std': 0.014854368447391955, 'prc_auc mean': 0.9538556716273254, 'prc_auc std': 0.02184670424726968}\n",
      "0.9153314073897096\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='ridge')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0125 {'roc_auc mean': 0.6261378409640445, 'roc_auc std': 0.06791685314693852, 'prc_auc mean': 0.574761882809699, 'prc_auc std': 0.06170898667695073}\n",
      "0.025 {'roc_auc mean': 0.6906008655978265, 'roc_auc std': 0.06069469298817478, 'prc_auc mean': 0.6026164819095869, 'prc_auc std': 0.05946246892443492}\n",
      "0.05 {'roc_auc mean': 0.7837215908535136, 'roc_auc std': 0.033088587931493794, 'prc_auc mean': 0.6793430629738084, 'prc_auc std': 0.060452380228749644}\n",
      "0.1 {'roc_auc mean': 0.8396918316692676, 'roc_auc std': 0.02282209002541289, 'prc_auc mean': 0.7315506437824587, 'prc_auc std': 0.04561536536349843}\n",
      "0.2 {'roc_auc mean': 0.8718682108584804, 'roc_auc std': 0.0168847671230538, 'prc_auc mean': 0.8022239786404531, 'prc_auc std': 0.02736454055128179}\n",
      "0.4 {'roc_auc mean': 0.8973939540444231, 'roc_auc std': 0.019900099229142286, 'prc_auc mean': 0.8397160684177871, 'prc_auc std': 0.023123942899067957}\n",
      "0.8 {'roc_auc mean': 0.9150688241055699, 'roc_auc std': 0.020264568789186338, 'prc_auc mean': 0.8789495853741849, 'prc_auc std': 0.036136536688061376}\n",
      "0.8034975882990182\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "for rate in rates:\n",
    "    score_dic = evaluate_classification_multi(X, rate, 20, model='rf')\n",
    "    print(rate, score_dic)\n",
    "    scores.append(score_dic['roc_auc mean'])\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
